1 # Copyright 2012 DWARF Debugging Information Format Committee
3 # All the little classes used in storing latex source data.
4 # Reads in the tex source and builds internal lists of the
5 # tokenized source. The tokenization is adequate
6 # for our purposes, but just barely adequate.
16 if isIndivid(c) == "y":
18 if ord(c) >= ord('a') and ord(c) <= ord('z'):
20 if ord(c) >= ord('A') and ord(c) <= ord('Z'):
22 # It is tex/latex, so backslash starts a word.
30 if isIndivid(c) == "y":
32 if ord(c) >= ord('a') and ord(c) <= ord('z'):
34 if ord(c) >= ord('A') and ord(c) <= ord('Z'):
36 if ord(c) >= ord('0') and ord(c) <= ord('9'):
38 # This is so we allow the colon in our tags
39 # Unfortunately, this gives trouble if we have a
40 # : at the end of a DW* name on input.
69 # self._tex DW\-\_ATE and the like
70 # self._underbar DW\_ATE and the like
71 # self._std the way a DW_ATE and the like looks in the standard
72 # self._label With all _ and - removed. Like DWATE
77 ind: a character taken as an individual character.
78 none: No characters seen yet.
79 shift: A character with the high bit of 8 bits set, not something we expect.
80 - In DW4 these high-bit-chars are special 3-character left and right quotes.
81 - charfix.py can replace these with Latex ascii quotes.
82 other: Some other character, but ascii, seemingly. """
88 # Class is "id", "ind","other","shift","none"
90 def insertid(self,string):
92 self._tex = list(string)
93 self._underbar = self._tex
95 self._label = self._tex
96 def setIndivid(self,c):
102 def setInitialIdChar(self,c):
105 def setNextIdChar(self,c):
108 def setInitialShift(self,c):
113 self._class = "shift"
114 def setNextShift(self,c):
116 self._underbar += [c]
119 self._class = "shift"
120 def setInitialOther(self,c):
125 self._class = "other"
126 def setNextOther(self,c):
128 self._underbar += [c]
131 self._class = "other"
132 def finishUpId(self):
133 """ This transforms the strings from the input form into
134 the internal forms we want.
141 while int(n) < len(self._tex):
143 if n < (len (self._tex) - 1) and c == "\\" and self._tex[n+1] == "-":
146 self._underbar += [c]
150 while int(n) < len(self._underbar):
151 c = self._underbar[n]
152 if n < (len (self._underbar) - 1) and c == "\\" and self._underbar[n+1] == "_":
159 while int(n) < len(self._std):
167 def dwprintquotedshortform(self,d):
168 print "'",self.shortform(d),"'",
169 def shortform(self,d):
172 if self._class == "ind":
174 self.dwprintquotedshortform(self._tex)
177 # This prints the token with end-line oddly.
179 self.dwprintquotedshortform(self._tex)
180 self.dwprintquotedshortform(self._underbar)
181 self.dwprintquotedshortform(self._std)
182 self.dwprintquotedshortform(self._label)
184 def dwwrite(self,outfile):
189 """using an input line, create a list of tokens for the line.
190 Legal class transitions in tokenize() are:
213 def tokenize(self,rec,filename,linenum):
214 """using an input line, create a list of tokens for the line.
215 Legal class transitions in tokenize() are:
231 print " Warning: encountered character ord:",ord(c), "at offset",charnum,"line",linenum,filename
232 if keepcomments == "d" and c == "%" and ( charnum == 0 or rec[charnum - 1] != "\\" ):
233 # Not keeping comments. We drop % and following to end of line
234 # unless preceeded by \
237 if c == "\n" or c == "\r":
238 # Just drop these for now. Allowing them
239 # would not be harmful.
241 elif dwclass == "none" or dwclass == "ind":
242 if isShift(c) == "y":
243 combotok.setInitialShift(c)
246 if isIndivid(c) == "y":
251 if isIdStart(c) == "y":
252 combotok.setInitialIdChar(c)
256 combotok.setInitialOther(c)
259 elif dwclass == "id":
260 if isIdNext(c) == "y":
261 combotok.setNextIdChar(c)
263 if isShift(c) == "y":
264 combotok.finishUpId()
265 self._toks += [combotok]
267 combotok.setInitialShift(c);
270 if isIndivid(c) == "y":
271 combotok.finishUpId()
272 self._toks += [combotok]
279 # Other class input, other starts here.
280 combotok.finishUpId()
281 self._toks += [combotok]
283 combotok.setInitialOther(c);
286 elif dwclass == "shift":
287 if isShift(c) == "y":
288 combotok.setNextShift(c);
290 if isIndivid(c) == "y":
291 self._toks += [combotok]
298 if isIdStart(c) == "y":
299 self._toks += [combotok]
301 combotok.setInitialIdChar(c);
304 # Shift class input, other starts here.
305 self._toks += [combotok]
307 combotok.setInitialOther(c);
310 elif dwclass == "other":
311 if isShift(c) == "y":
312 self._toks += [combotok]
314 combotok.setInitialShift(c);
317 if isIndivid(c) == "y":
318 self._toks += [combotok]
325 if isIdStart(c) == "y":
326 self._toks += [combotok]
328 combotok.setInitialIdChar(c);
331 combotok.setNextOther(c);
333 # Else case impossible.
335 #Finish up final non-empty other or id token
337 combotok.finishUpId()
338 self._toks += [combotok]
340 if dwclass == "shift":
341 self._toks += [combotok]
343 if dwclass == "other":
344 self._toks += [combotok]
346 def dwprint(self,linenum):
347 print "Number of tokens in line ",linenum," : ",len(self._toks)
348 if len(self._toks) == 0:
349 #Just print an empty line.
354 def dwwrite(self, outfile, linenum):
358 def dwtransformline(self,callfunc,myfile,lnum):
359 toks = callfunc(self._toks,myfile,lnum)
364 def __init__(self,name):
370 file = open(name,"r");
371 except IOError, message:
372 print >> sys.stderr , "File could not be opened: ", name
377 rec = file.readline()
385 aline.tokenize(rec,name,linenum)
386 self._lines += [aline]
389 print "Number of lines in ", self._name, ": ",len(self._lines)
391 for l in self._lines:
395 # The lnum is just for debugging messages.
397 outname = self._name + ".out"
400 outfile = open(outname,"w");
401 except IOError, message:
402 print >> sys.stderr , "Output File could not be opened: ", name
405 for l in self._lines:
406 l.dwwrite(outfile,lnum)
408 def dwtransformline(self,callfunc,myfile):
410 for l in self._lines:
411 l.dwtransformline(callfunc,myfile,lnum)
421 def addFile(self,name):
426 print "Number of files: ",len(self._files);
427 for f in self._files:
430 for f in self._files:
432 def dwtransformline(self,callfunc):
433 for f in self._files:
434 f.dwtransformline(callfunc,f)
437 def setkeepordeletecomments(val):
438 """ Pass in "k" or "d" to keep or delete comments, respectively """
442 def readFilelist(filelist):