Coverage for linuxpy/codegen/base.py: 0%

2# This file is part of the linuxpy project

5# Distributed under the GPLv3 license. See LICENSE for more info.

8import datetime

9import logging

10import os

11import pathlib

12import platform

13import re

14import subprocess

15import tempfile

16import textwrap

17import xml.etree.ElementTree

19CTYPES_MAP = {

20 "char*": "ccharp",

21 "unsigned char": "u8",

22 "signed char": "cchar",

23 "char": "cchar",

24 "short unsigned int": "u16",

25 "short int": "i16",

26 "unsigned int": "cuint",

27 "int": "cint",

28 "long unsigned int": "culong",

29 "long long unsigned int": "culonglong",

30 "long int": "clong",

31 "long long int": "clonglong",

32 "void *": "cvoidp",

33 "void": "None",

34}

36MACRO_RE = re.compile(r"#define[ \t]+(?P<name>[\w]+)[ \t]+(?P<value>.+)\s*")

39class CEnum:

40 def __init__(self, name, prefixes, klass=None, with_prefix=False, filter=lambda _n, _v: True):

41 self.name = name

42 if isinstance(prefixes, str):

43 prefixes = [prefixes]

44 self.prefixes = prefixes

45 if klass is None:

46 klass = "IntFlag" if any("FLAG" in prefix for prefix in prefixes) else "IntEnum"

47 self.klass = klass

48 self.with_prefix = with_prefix

49 self.values = None

50 self.filter = filter

52 def add_item(self, name, value):

53 if self.empty:

54 self.values = []

55 self.values.append((name, value))

57 @property

58 def empty(self):

59 return self.values is None

61 def __repr__(self):

62 if self.values:

63 fields = "\n".join(f" {name} = {value}" for name, value in self.values)

64 else:

65 fields = " pass"

66 return f"""\

67class {self.name}(enum.{self.klass}):

68{fields}

69"""

72class CStruct:

73 def __init__(self, node, name, node_members, pack=False):

74 self.node = node

75 self.node_members = node_members

76 self.pack = pack

77 self.name = name

78 self.parent = None

79 self.fields = []

80 self.children = {}

81 self.is_anonymous = not self.name

82 self.anonymous_fields = []

84 @property

85 def type(self):

86 return self.node.tag

88 @property

89 def id(self):

90 return self.node.get("id")

92 @property

93 def member_ids(self):

94 return self.node.get("members").split()

96 @property

97 def context_id(self):

98 return self.node.get("context")

100 @property

101 def class_text(self):

102 text = f"class {self.name}({self.type}):\n"

103 if self.pack:

104 text += " _pack_ = True\n"

105 if self.children:

106 children = "\n".join(str(child) for child in self.children.values())

107 text += textwrap.indent(children, 4 * " ")

108 text += "\n"

109 if self.anonymous_fields:

110 text += f" _anonymous_ = {tuple(self.anonymous_fields)}\n"

111 if not any((self.pack, self.children, self.anonymous_fields)):

112 text += " pass"

113 return text

114

115 @property

116 def fields_text(self):

117 fields = ", ".join(f'("{fname}", {ftype})' for fname, ftype in self.fields)

118 return f"{self.name}._fields_ = [{fields}]"

119

120 def __repr__(self):

121 return f"{self.class_text}\n{self.fields_text}\n"

122

123

124def lines(filename):

125 with open(filename) as source:

126 for line in source:

127 yield line.strip()

128

129

130def macro_lines(filename):

131 for line in lines(filename):

132 if line.startswith("#define"):

133 matches = MACRO_RE.match(line)

134 if matches:

135 data = matches.groupdict()

136 yield data["name"], data["value"]

137

138

139def decode_macro_value(value, context, name_map):

140 value = value.replace("*/", "")

141 value = value.replace("/*", "#")

142 value = value.replace("struct ", "")

143 value = value.replace("union ", "")

144 value = value.replace("(1U", "(1")

145 value = value.strip()

146 for dtype in "ui":

147 for size in (8, 16, 32, 64):

148 typ = f"{dtype}{size}"

149 value = value.replace(f"__{typ}", typ)

150 for ctype, pytype in CTYPES_MAP.items():

151 value = value.replace(" " + ctype, pytype)

152 try:

153 return f"0x{int(value):X}"

154 except ValueError:

155 try:

156 return f"0x{int(value, 16):X}"

157 except ValueError:

158 for c_name, py_name in name_map.items():

159 py_class, name = py_name.split(".", 1)

160 if py_class == context.name:

161 py_name = name

162 value = value.replace(c_name, py_name)

163 return value

164

165

166def find_macro_enum(name, enums):

167 for cenum in enums:

168 for prefix in cenum.prefixes:

169 if prefix in name:

170 return cenum, prefix

171

172

173def fill_macros(filename, name_map, enums):

174 for cname, cvalue in macro_lines(filename):

175 if "PRIVATE" in cname:

176 continue

177 cenum = find_macro_enum(cname, enums)

178 if cenum is None:

179 continue

180 cenum, prefix = cenum

181 if not cenum.filter(cname, cvalue):

182 continue

183 py_value = decode_macro_value(cvalue, cenum, name_map)

184 py_name = cname[0 if cenum.with_prefix else len(prefix) :]

185 if py_name[0].isdigit():

186 py_name = f"_{py_name}"

187 cenum.add_item(py_name, py_value)

188 name_map[cname] = f"{cenum.name}.{py_name}"

189

190

191def find_xml_base_type(etree, context, type_id):

192 while True:

193 node = etree.find(f"*[@id='{type_id}']")

194 if node is None:

195 return

196 if node.tag == "Struct":

197 return node.get("name"), node.get("id")

198 elif node.tag == "FundamentalType":

199 return CTYPES_MAP[node.get("name")], node.get("id")

200 elif node.tag == "PointerType":

201 name, base_id = find_xml_base_type(etree, context, node.get("type"))

202 return f"POINTER({name})".format(name), base_id

203 elif node.tag == "Union":

204 return node.get("name"), node.get("id")

205 elif node.tag == "ArrayType":

206 name, base_id = find_xml_base_type(etree, context, node.get("type"))

207 if name == "u8":

208 name = "cchar"

209 nmax = node.get("max")

210 if nmax:

211 n = int(nmax) + 1

212 return f"{name} * {n}", base_id

213 else:

214 return f"POINTER({name})", base_id

215

216 type_id = node.get("type")

217

218

219def get_structs(header_filename, xml_filename, decode_name):

220 etree = xml.etree.ElementTree.parse(xml_filename)

221 header_tag = etree.find(f"File[@name='{header_filename}']")

222 struct_map = {}

223 structs = []

224 if header_tag is None:

225 return structs

226 header_id = header_tag.get("id")

227 nodes = [node for node in etree.findall(f"*[@file='{header_id}']") if node.tag in {"Union", "Struct"}]

228 for node in nodes:

229 members = node.get("members")

230 if members is not None:

231 member_ids = node.get("members").split()

232 else:

233 # Handle empty structure case

234 member_ids = []

235 fields = (etree.find(f"*[@id='{member_id}']") for member_id in member_ids)

236 fields = [field for field in fields if field.tag not in {"Union", "Struct", "Unimplemented"}]

237 align = node.get("align")

238 if align is None:

239 pack = False

240 else:

241 pack = int(align) == 8

242 name = node.get("name")

243 if name:

244 name = decode_name(name)

245 struct = CStruct(node, name, fields, pack)

246 struct_map[struct.id] = struct

247 structs.append(struct)

248 for struct in structs:

249 if struct.context_id != "_1":

250 parent = struct_map.get(struct.context_id)

251 if parent:

252 parent.children[struct.id] = struct

253 struct.parent = parent

254 else:

255 logging.error("Could not find parent")

256 if not struct.name and struct.parent:

257 struct.name = f"M{len(struct.parent.children)}"

258 for struct in structs:

259 for node in struct.node_members:

260 name = node.get("name")

261 base = find_xml_base_type(etree, struct, node.get("type"))

262 if base is None:

263 logging.warning("unknown field for %s", struct.name)

264 else:

265 base_type, base_id = base

266 child = struct.children.get(base_id)

267 if child is not None:

268 if not name:

269 name = child.name.lower()

270 struct.anonymous_fields.append(name)

271 if not base_type:

272 base_type = f"{struct.name}.{child.name}"

273 struct.fields.append((name, base_type))

274

275 return structs

276

277

278def cname_to_pyname(

279 name: str,

280 capitalize=True,

281 splitby="_",

282):

283 if name.startswith("v4l2_"):

284 name = name[5:]

285 if capitalize:

286 name = name.capitalize()

287 return "".join(map(str.capitalize, name.split(splitby)))

288

289

290def get_enums(header_filename, xml_filename, enums, decode_name):

291 etree = xml.etree.ElementTree.parse(xml_filename)

292 header_tag = etree.find(f"File[@name='{header_filename}']")

293 structs = {}

294 if header_tag is None:

295 return structs

296 header_id = header_tag.get("id")

297 nodes = etree.findall(f"Enumeration[@file='{header_id}']")

298 for node in nodes:

299 cname = node.get("name")

300 if not cname:

301 continue

302 py_name = cname_to_pyname(decode_name(cname))

303 prefix = cname.upper() + "_"

304 if len(node) > 1:

305 raw_names = [child.get("name") for child in node]

306 common_prefix = os.path.commonprefix(raw_names)

307 else:

308 common_prefix = ""

309 values = []

310 for child in node:

311 name = child.get("name")

312 name = name.removeprefix(prefix)

313 name = name.removeprefix(common_prefix)

314 if "PRIVATE" in name:

315 continue

316 if name[0].isdigit():

317 name = f"_{name}"

318 value = int(child.get("init"))

319 values.append((name, value))

320 enum = CEnum(py_name, prefix)

321 enum.values = values

322 enums.append(enum)

323

324

325def code_format(text, filename):

326 try:

327 cmd = ["ruff", "check", "--fix", "--stdin-filename", str(filename)]

328 result = subprocess.run(cmd, capture_output=True, check=True, text=True, input=text)

329 fixed_text = result.stdout

330 except Exception as error:

331 print(repr(error))

332 fixed_text = text

333

334 try:

335 cmd = ["ruff", "format", "--stdin-filename", str(filename)]

336 result = subprocess.run(cmd, capture_output=True, check=True, text=True, input=fixed_text)

337 fixed_text = result.stdout

338 except Exception as error:

339 print(repr(error))

340 return fixed_text

341

342

343def nullf(x):

344 return x

345

346

347def run(name, headers, template, macro_enums, output=None, decode_struct_name=nullf, decode_enum_name=nullf):

348 cache = {}

349 temp_dir = tempfile.mkdtemp()

350 logging.info("Starting %s...", name)

351 structs = []

352 for header in headers:

353 logging.info(" Building %s for %s...", header, name)

354 fill_macros(header, cache, macro_enums)

355 base_header = os.path.split(os.path.splitext(header)[0])[1]

356 xml_filename = os.path.join(temp_dir, base_header + ".xml")

357 cmd = f"castxml --castxml-output=1.0.0 -o {xml_filename} {header}"

358 assert os.system(cmd) == 0

359 structs += get_structs(header, xml_filename, decode_name=decode_struct_name)

360

361 get_enums(header, xml_filename, macro_enums, decode_name=decode_enum_name)

362

363 structs_definition = "\n\n".join(struct.class_text for struct in structs if struct.parent is None)

364 structs_fields = "\n".join(struct.fields_text for struct in structs if struct.parent is None)

365

366 structs_body = "\n\n".join(str(struct) for struct in structs if struct.parent is None)

367 enums_body = "\n\n".join(str(enum) for enum in macro_enums if "IOC" not in enum.name)

368 iocs_body = "\n\n".join(str(enum) for enum in macro_enums if "IOC" in enum.name)

369

370 fields = {

371 "name": name,

372 "date": datetime.datetime.now(),

373 "system": platform.system(),

374 "release": platform.release(),

375 "version": platform.version(),

376 "enums_body": enums_body,

377 "structs_body": structs_body,

378 "structs_definition": structs_definition,

379 "structs_fields": structs_fields,

380 "iocs_body": iocs_body,

381 }

382 text = template.format(**fields)

383 logging.info(" Applying ruff to %s...", name)

384 text = code_format(text, output)

385 logging.info(" Writting %s...", name)

386 if output is None:

387 print(text, end="", flush=True)

388 else:

389 output = pathlib.Path(output)

390 with output.open("w") as fobj:

391 print(text, end="", flush=True, file=fobj)

392 logging.info("Finished %s!", name)