Merge branch 'HEAD' of dwarfgit@dwarfstd.org:dwarf-doc.git
[dwarf-doc.git] / dwarf5 / tools / refer.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex(+) to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12 # This version assumes relevant commands are used
13 # all-on-one-line, not with operands on successive lines.
14
15 # This is the simplest 'parse' of the .tex that we can manage
16 # while still finding what we want to find.
17 # One would hardly call it a parser, really.
18
19 # There are essentially three namespaces at present in the document.
20 # The hyperlink/hypertarget namespace.
21 # The label  vref ref namespace
22 #   which also involves  our refersec referfol.
23 # The index namespace, which we are not presently filling in very much. 
24
25 import sys
26 import os
27 import os.path
28 import fileio
29
30 # These two hold the commands we care about so we can
31 # bypass most lines easily in phase two.
32 # All the newdwfnamecommands
33 global dwfnamecommsdict
34 # All the newcommand instances.
35 global newcommsdict
36
37 # NAME suffix. No index, just text shows.
38 # Not very useful so far
39 global namedict
40
41
42 # Targets for \hypertarget and \hyperlink
43 #targhyperdict is all the \hyperarget instances.
44 #linkhyperdict is all the \hyperlink instances.
45 global targhyperdict
46 global linkhyperdict
47
48 # label targets (labels)
49 global labeldict
50 # \refersec \ref \vref to labels
51 global labelrefdict
52
53 # The index content for named things.
54 # The strings here are the links, the targets are
55 # built by the latex index software.
56 # So this dictionary is not really needed here (yet)
57 # and is not yet fully built up.
58 global indexdict
59
60
61 newcommsdict = {}
62 dwfnamecommsdict = {}
63
64 # Links meaning \livelink \livetarg \livetargi macros
65 #linksdict  = {}
66 #targsdict  = {}
67 namedict  = {}
68 targhyperdict= {}
69 linkhyperdict= {}
70 labeldict = {}
71 labelrefdict = {}
72 indexdict = {}
73
74 global linestoignore
75 linestoignore = []
76
77 # lines_to_ignore is a terrible hack.
78 def add_lines_to_ignore(myfile,lowline,highline):
79   global linestoignore
80   linestoignore += [(myfile._name,lowline,highline)]
81 def in_lines_to_ignore(myfile,line):
82   global linestoignore
83   n = myfile._name
84   for x in linestoignore:
85     (n,l,h) = x
86     if  myfile._name != n:
87       continue
88     if line < l:
89       continue
90     if line >h:
91       continue
92     return "y"
93   return "n"
94    
95
96 # a list of words to ignore: silly stuff.
97 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
98 "Standards":0,"Committee":0,"Version":0 }
99
100 class tokmention:
101   # _hassubcomm,_target are so we can know a command has side effects
102   # of a hyperlink or a livelinki.
103   def __init__(self):
104     self._token = '' 
105     self._file = ""
106     self._line = 0
107     self._hassubcomm = "n"
108     self._subcomm = ""
109     self._target = ""
110   def __init__(self,tok,filename,line):
111     self._token = tok
112     self._file = filename
113     self._line = line
114     self._hassubcomm = "n"
115     self._subcomm = ""
116     self._target = ""
117   def settarget(self,subcom,target):
118     self._hassubcomm = "y"
119     self._subcomm = subcom
120     self._target = target
121   def dumptm(self):
122     print "Tokmention token  class ",self._token._class,"name",self._token._tex
123     print "Tokmention file,lne ",self._file._name,self._line
124     print "Tokmention subcomm ",self._hassubcomm,self._subcomm,self._target
125
126 def isid(tok):
127    if tok._class != "id":
128       return "n"
129    return "y"
130
131 def ischar(tok,c):
132    if tok._class != "ind":
133       return "n"
134    if len(tok._tex) != 1:
135        return "n"
136    if tok._tex[0] != c:
137        return "n"
138    return "y"
139
140 def dwspace(tok):
141   if ischar(tok," ") == "y":
142     return "y"
143   if ischar(tok,"\t") == "y":
144     return "y"
145   return "n"
146   
147   
148 def isbrace(tok,brace):
149   if tok._class != "ind":
150      return "n"
151   if len(tok._tex) != 1:
152      return "n"
153   if brace == tok._tex[0]:
154      return "y"
155   return "n"
156
157 def toknamestring(t):
158   """ Turn a token into its string as a string """
159   return ''.join(t._tex)
160
161
162 def pickup(linetoks,tnumin,pattern,myfile,linenum,suppresserr):
163   """ The token pattern characters are
164   i meaning identifier
165   e meaning identifier, but ifnext token is }
166      we construct an empty identifier for it.
167   [space] meaning whitespace
168   { meaning left brace
169   } meaning right brace
170   * meaning any token except } and end-line
171   
172   Precondition:  linetoks[tnumin] is identifier (meaning a command)
173   Returns: a token list, one per non-space in the pattern.
174      For the *, the token is itself a list of whatever it contains.
175   """
176   outtoks = []
177   numabsorbed = 1
178   inlen = len(linetoks) 
179   curnum = tnumin
180   curtok = linetoks[curnum]
181   patterncharnum = -1
182   for c in pattern:
183     patterncharnum = patterncharnum + 1
184     if curnum >= inlen:
185       if suppresserr == "n":
186         print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
187       return outtoks,numabsorbed
188     curtok = linetoks[curnum]
189     if c == " ":
190       while dwspace(curtok) == "y":
191         curnum = curnum + 1
192         if curnum >= inlen:
193           if suppresserr == "n":
194             print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
195           return outtoks,numabsorbed
196         numabsorbed = numabsorbed + 1
197         curtok = linetoks[curnum]
198       continue
199     elif c == "i":
200       if curtok._class != "id":
201         if suppresserr == "n":
202           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
203         return outtoks,numabsorbed
204       numabsorbed = numabsorbed + 1
205       outtoks += [curtok]
206       curnum = curnum + 1
207       continue
208     elif c == "e":
209       if curtok._class != "id":
210         if isbrace(curtok,"}") == "y":
211           tk = fileio.dwtoken()
212           tk.insertid("",linenum)
213           outtoks += [tk]
214           # Do not update location.
215           continue
216         else:
217           if suppresserr == "n":
218             print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
219           return outtoks,numabsorbed
220       else: 
221         numabsorbed = numabsorbed + 1
222         outtoks += [curtok]
223         curnum = curnum + 1
224       continue
225     elif c == "{":
226       if isbrace(curtok,"{")  == "y":
227         outtoks += [curtok]
228         curnum = curnum + 1
229         numabsorbed = numabsorbed + 1
230       else:
231         if suppresserr == "n":
232           print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
233         return outtoks,numabsorbed
234     elif c == "}":
235       if isbrace(curtok,"}")  == "y":
236         outtoks += [curtok]
237         curnum = curnum + 1
238         numabsorbed = numabsorbed + 1
239       else:
240         if suppresserr == "n":
241           print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
242         return outtoks,numabsorbed
243     elif c == "[":
244       if isbrace(curtok,"[")  == "y":
245         outtoks += [curtok]
246         curnum = curnum + 1
247         numabsorbed = numabsorbed + 1
248       else:
249         if suppresserr == "n":
250           print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
251         return outtoks,numabsorbed
252     elif c == "]":
253       if isbrace(curtok,"]")  == "y":
254         outtoks += [curtok]
255         curnum = curnum + 1
256         numabsorbed = numabsorbed + 1
257       else:
258         if suppresserr == "n":
259           print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
260         return outtoks,numabsorbed
261     elif c == "*":
262       outlist = []
263       curtok = linetoks[curnum]
264       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
265         if dwspace(curtok) == "n":
266            outlist += [curtok]
267         curnum = curnum + 1
268         if curnum >= inlen:
269           outtoks += [outlist]
270           if patterncharnum < (len(pattern) -1): 
271             if suppresserr == "n":
272               print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
273           return outtoks,numabsorbed
274         numabsorbed = numabsorbed + 1
275         curtok = linetoks[curnum]
276       # Found a right brace, so done here.
277       outtoks += [outlist]
278     else:
279         if suppresserr == "n":
280           print "ERROR pattern had unexpected character ",pattern
281   return outtoks,numabsorbed
282
283 def printbadcommand(name,myfile,myline):
284   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
285
286 def applytodict(d,k,v):
287   keystring = k
288   if d.has_key(keystring) == 0:
289      d[keystring] =  [v]
290   else:
291      # This is a duplicate entry.
292      # We will report on it later as appropriate.
293      existing = d.get(keystring)
294      existing += [v]
295      d[keystring] =  existing
296
297 # See how many "{" there are on the line.
298 # return the count.
299 def countbraces(linetoks,tnumin):
300   lasttoknum = len(linetoks) -1
301   lb = 0
302   while tnumin < lasttoknum:
303     x = linetoks[tnumin] 
304     if x._class == "ind":
305        n = toknamestring(x)
306        if n == "{":
307           lb = lb + 1
308     tnumin = tnumin + 1
309   return lb
310      
311
312
313 # Here we try two different parses, the [] is optional
314 # with simplenametable.
315 def processbegin(linetoks,tnumin,myfile,linenum):
316   global targhyperdict
317   global indexdict
318   lbracecount = countbraces(linetoks,tnumin)
319   if lbracecount < 3:
320     return 1
321   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum,"y")
322   if len(ourtoks) == 13:
323     lcom = ourtoks[2]
324     lcomname = toknamestring(lcom)
325     if lcomname != "simplenametable":
326       return inlen
327     targ = ourtoks[11]
328     hypstr = toknamestring(targ)
329     hypmen = tokmention(targ,myfile,linenum)
330     applytodict(labeldict,hypstr,hypmen)
331     return inlen
332   ourtoks,inlen = pickup(linetoks,tnumin," i { i } { * } { i }",myfile,linenum,"y")
333   if len(ourtoks) < 10:
334       return inlen
335   lcom = ourtoks[2]
336   lcomname = toknamestring(lcom)
337   if lcomname != "simplenametable":
338       return inlen
339   targ = ourtoks[8]
340   hypstr = toknamestring(targ)
341   hypmen = tokmention(targ,myfile,linenum)
342   applytodict(labeldict,hypstr,hypmen)
343   return inlen
344
345 # If justlink == "y" this is a hypertarget command literally.
346 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
347   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
348   global targhyperdict
349   global indexdict
350   t = linetoks[tnumin]
351   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
352   if len(ourtoks) > 5:
353     t2 = ourtoks[2];
354     index = tokmention(t2,myfile,linenum)
355     name = toknamestring(t2)
356     applytodict(targhyperdict,name,index)
357     if justlink == "n":
358       t2 = ourtoks[5];
359       # Ignore for now.
360       #name = toknamestring(t2)
361       #if len(name) > 0:
362       #  index = tokmention(t2,myfile,linenum)
363       #  applytodict(indexdict,name,index)
364   else:
365     tn = toknamestring(linetoks[tnumin])
366     printbadcommand(tn,myfile,linenum)
367   return inlen
368 def livetargiprocess(linetoks,tnumin,myfile,linenum):
369   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
370   global targhyperdict
371   global indexdict
372   t = linetoks[tnumin]
373   # The first * here used to be e but that was insufficiently general.
374   # The index is generated from the entire  * (token list 8) , not from the
375   # initial  *, token list 5
376   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * } { * }",myfile,linenum,"n")
377   if len(ourtoks) > 5:
378     t2 = ourtoks[2];
379     index = tokmention(t2,myfile,linenum)
380     name = toknamestring(t2)
381     applytodict(targhyperdict,name,index)
382
383     # WARNING: looking at 5 this way was wrong.
384     #t2 = ourtoks[5];
385     #name = toknamestring(t2)
386     #if len(name) > 0:
387     #  index = tokmention(t2,myfile,linenum)
388     #  applytodict(indexdict,name,index)
389   else:
390     tn = toknamestring(linetoks[tnumin])
391     printbadcommand(tn,myfile,linenum)
392   return inlen
393 # if justlink == "y" it is a plain hyperlinkcommand
394 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
395   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
396   global linkhyperdict
397   global indexdict
398   t = linetoks[tnumin]
399   if justlink == "y":
400     ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
401     if len(ourtoks) > 5:
402       t2 = ourtoks[2];
403       index = tokmention(t2,myfile,linenum)
404       name = toknamestring(t2)
405       applytodict(linkhyperdict,name,index)
406     else:
407       tn = toknamestring(linetoks[tnumin])
408       printbadcommand(tn,myfile,linenum)
409   else:
410     ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
411     if len(ourtoks) == 4:
412       t2 = ourtoks[2];
413       index = tokmention(t2,myfile,linenum)
414       name = toknamestring(t2)
415       applytodict(linkhyperdict,name,index)
416     else:
417       tn = toknamestring(linetoks[tnumin])
418       printbadcommand(tn,myfile,linenum)
419   return inlen
420 def labelprocess(linetoks,tnumin,myfile,linenum):
421   """ \label{alabel} """
422   global labeldict
423   t = linetoks[tnumin]
424   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
425   if len(ourtoks) > 2:
426     t2 = ourtoks[2];
427     index = tokmention(t2,myfile,linenum)
428     name = toknamestring(t2)
429     applytodict(labeldict,name,index)
430   else:
431     tn = toknamestring(linetoks[tnumin])
432     printbadcommand(tn,myfile,linenum)
433
434   return inlen
435 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
436   """ \addtoindex{strings} """
437   global indexdict
438   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
439   if len(ourtoks) > 2:
440     # The * means a list of tokens.
441     fake = ""
442     #t2 = ourtoks[2];
443     #index = tokmention(t2,myfile,linenum)
444     #name = toknamestring(t2)
445     #applytodict(indexdict,name,index)
446   else:
447     tn = toknamestring(linetoks[tnumin])
448     printbadcommand(tn,myfile,linenum)
449   return inlen
450 def hyperlinkname(name,tnumin,myfile,linenum):
451   global linkhyperdict
452   tkmod = fileio.dwtoken()
453   tkmod.insertid(name,linenum)
454   tm = tokmention(tkmod,myfile,linenum)
455   applytodict(linkhyperdict,name,tm)
456   return 1
457 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
458   """ \hyperlink{entryname} """
459   global linkhyperdict
460   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
461   if len(ourtoks) > 2:
462     t2 = ourtoks[2];
463     index = tokmention(t2,myfile,linenum)
464     name = toknamestring(t2)
465     applytodict(linkhyperdict,name,index)
466   else:
467     tn = toknamestring(linetoks[tnumin])
468     printbadcommand(tn,myfile,linenum)
469   return inlen
470
471 def indexprocess(linetoks,tnumin,myfile,linenum):
472   """ \index{indexentryname} """
473   global indexdict
474   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
475   if len(ourtoks) > 2:
476     fake = ""
477     # For now not bothering with index strings
478     #t2 = ourtoks[2];
479     #index = tokmention(t2,myfile,linenum)
480     #name = toknamestring(t2)
481     #applytodict(indexdict,name,index)
482   else:
483     tn = toknamestring(linetoks[tnumin])
484     printbadcommand(tn,myfile,linenum)
485   return inlen
486 def refersecprocess(linetoks,tnumin,myfile,linenum):
487   """ \refersec{label} """
488   global labelrefdict
489   t = linetoks[tnumin]
490   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
491   if len(ourtoks) > 2:
492     t2 = ourtoks[2];
493     index = tokmention(t2,myfile,linenum)
494     name = toknamestring(t2)
495     applytodict(labelrefdict,name,labelrefdict)
496   else:
497     tn = toknamestring(linetoks[tnumin])
498     printbadcommand(tn,myfile,linenum)
499   return inlen
500
501 def firstnonblank(linetoks):
502   tnum = 0
503   lasttoknum = len(linetoks)
504   while tnum < lasttoknum:
505     x = linetoks[tnum]
506     if x._class != "ind":
507       return tnum
508     if toknamestring(x) == " ":
509       tnum = tnum + 1
510       continue
511     elif toknamestring(x) == "\t":
512       tnum = tnum + 1
513       continue
514     return tnum 
515   return tnum
516
517 # return tuple of "y" and token num
518 # or "n" and 0
519 def nextnonblank(linetoks,tnum):
520   lasttoknum = len(linetoks)
521   tnum = tnum + 1
522   while tnum < lasttoknum:
523     x = linetoks[tnum]
524     if x._class != "ind":
525       return ("y",tnum)
526     if toknamestring(x) == " ":
527       tnum = tnum + 1
528       continue
529     elif toknamestring(x) == "\t":
530       tnum = tnum + 1
531       continue
532     return ("y",tnum)
533   return ("n",0)
534
535 # Return "n" or "y". If "y" add the proper tnum of 5 and 7.
536 # Even where the token list has spaces.
537 # Special examples, several of things like this exist:
538   # \newcommand{\CLASSaddress}      {\livelinki{chap:address}{address}{address class}}
539   #  \newcommand{\DWOPbregtwo}{\hyperlink{chap:DWOPbregn}{DW\_OP\_breg2}}
540   #    0        1   2        34   5      6   7
541 def speciallinkdef(linetoks,tnum):
542   if len(linetoks) < (int(tnum) +7):
543     return ("n",0,0)
544   (ok,tn1) = nextnonblank(linetoks,tnum)
545   if ok == "n":
546     return ("n",0,0)
547   (ok,tn2) = nextnonblank(linetoks,tn1)
548   if ok == "n":
549     return ("n",0,0)
550   (ok,tn3) = nextnonblank(linetoks,tn2)
551   if ok == "n":
552     return ("n",0,0)
553   (ok,tn4) = nextnonblank(linetoks,tn3)
554   if ok == "n":
555     return ("n",0,0)
556   (ok,tn5) = nextnonblank(linetoks,tn4)
557   if ok == "n":
558     return ("n",0,0)
559   (ok,tn6) = nextnonblank(linetoks,tn5)
560   if ok == "n":
561     return ("n",0,0)
562   (ok,tn7) = nextnonblank(linetoks,tn6)
563   if ok == "n":
564     return ("n",0,0)
565   t5 = linetoks[tn5]
566   if toknamestring(t5) == "\\livelinki":
567     return ("y",tn5,tn7)
568   elif  toknamestring(t5) == "\\hyperlink":
569     return ("y",tn5,tn7)
570   return ("n",0,0)
571
572 # Deals solely with finding new commands.
573 # This done as a first pass so we can recognize when tokens are
574 # really commands, something transfunc2, the second pass, 
575 # wants to know. 
576 # Special examples, several of things like this exist:
577   # \newcommand{\CLASSaddress}      {\livelinki{chap:address}{address}{address class}}
578   #  \newcommand{\DWOPbregtwo}{\hyperlink{chap:DWOPbregn}{DW\_OP\_breg2}}
579   #    0        1   2        34   5      6   7
580
581 def transfunc1(linetoks,myfile,linenum):
582   global dwfnamecommsdict
583   global newcommsdict
584
585   if len(linetoks) < 1:
586     return linetoks
587   tnum = firstnonblank(linetoks)
588   if tnum >= len(linetoks):
589     return linetoks
590   initialtok = linetoks[tnum]
591   itokstring=toknamestring(initialtok)
592   if itokstring == "\\expandafter\\def\\csname":
593     return linetoks
594   if in_lines_to_ignore(myfile,linenum) == "y":
595     return linetoks
596   if itokstring == "\\newcommand":
597     t1 = linetoks[tnum+1]
598     if not isbrace(t1,'{'):
599       print "Improper character in newcommand", myfile,linenum
600       sys.exit(1)
601     t2 = linetoks[tnum+2]
602     if toknamestring(t2) == "\\simplenametablerule":
603        add_lines_to_ignore(myfile,linenum,linenum+18)
604
605     # If the content is a hyperlink with a real id as the hyperlink
606     # target, make it special.
607     # Because some of the special lines have blanks, we let speciallink
608     # find the right indices for our tokens.
609     (res,tn5,tn7) = speciallinkdef(linetoks,tnum)
610     if res == "y":
611       t5 = linetoks[tn5]
612       t7 = linetoks[tn7]
613       tm = tokmention(t5,myfile,linenum);
614       tm.settarget(toknamestring(t5),toknamestring(t7))
615       applytodict(newcommsdict,toknamestring(t2),tm)
616     elif toknamestring(t2) != "\\newdwfnamecommands":
617        tm = tokmention(t2,myfile,linenum)
618        applytodict(newcommsdict,toknamestring(t2),tm)
619     #Be silent on "newcommand { newdwfnamecommands }...", it is not relevant here.
620     return linetoks
621   elif itokstring == "\\newdwfnamecommands":
622     t1 = linetoks[tnum+1]
623     if not isbrace(t1,'{'):
624        print "Improper character in newdwfnamecommands", myfile._name,linenum
625        sys.exit(1)
626     # The token name string will be DWsomething and we want
627     # The token to appear as \DWsomething as that is how references
628     # The usages determine what secondary actions are applied.
629     # are coded.
630     t2 = linetoks[tnum+2]
631     tkmod = fileio.dwtoken()
632     tkmod.insertid("\\" + toknamestring(t2),linenum)
633     tm = tokmention(tkmod,myfile,linenum)
634     applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
635     return linetoks
636   return linetoks
637
638
639 def delsuffix(n,suf):
640   slen = len(suf)
641   nlen = len(n)
642   lastcharnum = nlen - slen
643   outstring = n[0:lastcharnum]
644   return outstring
645 def deloptionalprefix(n,pref):
646   if not n.startswith(pref):
647     return n
648   plen = len(pref)
649   nlen = len(n)
650   outstring = n[plen:nlen]
651   return outstring
652
653 def printodderr(rawname,comname,myfile,linenum):
654   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
655
656 def rawnameiscommand(t,suff):
657    if not t.startswith("\\DW"):
658      return ""
659    if not t.endswith(suff):
660      return ""
661    commandname = delsuffix(t,suff)
662    return commandname
663
664 # Delete any leading backslash.
665 # Prefix the result with chap:
666 def makelinkname(t):
667    s = deloptionalprefix(t,"\\");
668    s2 = "chap:" + s
669    return s2;
670    
671
672 # Assumes all new commands known already.
673 # This deals with targets and links (various flavors).
674 def transfunc2(linetoks,myfile,linenum):
675   global newcommsdict
676   global dwfnamecommsdict
677   global newcommsdict
678
679   # Link naming target
680   global linkhyperdict
681   # TARG suffix
682   global targhyperdict
683   # INDX suffix
684   global indexdict
685   # NAME suffix
686   global namedict
687
688   if len(linetoks) < 1:
689     return linetoks
690   if in_lines_to_ignore(myfile,linenum) == "y":
691     return linetoks
692   initialtok = linetoks[0]
693   itokstring=toknamestring(initialtok)
694   # Skip all the newcommand stuff.
695   if itokstring == "\\newcommand":
696     return linetoks
697   elif itokstring == '\\newdwfnamecommands':
698     return linetoks
699
700   # Now deal with a regular line.
701
702   tnumin = 0
703   changes = 0
704   lasttoknum = len(linetoks) -1
705   for x in linetoks:
706     if int(tnumin) > int(lasttoknum):
707       break
708     t = linetoks[tnumin]
709     if t._class != "id":
710       tnumin = tnumin + 1
711       continue
712     rawname = toknamestring(t)
713     commandname=""
714     #rawnameiscommand(rawname,"",basecommand)
715     if rawname == "\\expandafter\\def\\csname":
716       return linetoks
717     if rawname == "\\begin":
718       tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
719       tnumin = tnumin + tnumcount
720       continue
721     if dwfnamecommsdict.has_key(rawname):
722       # We know this one. 
723       # It is a default case name reference.
724       # index the DWname
725       # Link is to chap:DWname
726       tm = tokmention(t,myfile,linenum)
727       linkname = makelinkname(rawname)
728       indxname = deloptionalprefix(commandname,"\\")
729       applytodict(indexdict,indxname,tm);
730
731       applytodict(linkhyperdict,linkname,tm);
732       tnumin = tnumin + 1
733       continue
734     if  newcommsdict.has_key(rawname):
735       # We know this one. We have to see what it is
736       # To decide what to do.
737       # some DWOPbreg*  DWOPreg*   and MDfive are special.
738       # Some commands are really hyperlinks.
739       # A variety of other such defined commands are irrelevant to us here.
740
741       tnumcount = 1
742       tml = newcommsdict[rawname]
743       if len(tml) != 1:
744          print "Too many newcommsdict entries for ",rawname,len(tml)
745          tml[0].dumptm()
746          tml[1].dumptm()
747          sys.exit(1)
748       tm = tml[0]
749       if tm._hassubcomm == "y":
750         com = tm._subcomm
751         targname = tm._target
752         com = tm._subcomm
753         targname = tm._target
754         if com == "\\hyperlink":
755           t2 = fileio.dwtoken()
756           t2.insertid(targname,linenum)
757           tm2 = tokmention(t2,myfile,linenum)
758           name = toknamestring(t2)
759           applytodict(linkhyperdict,targname,tm2)
760         elif com == "\\livelinki":
761           t2 = fileio.dwtoken()
762           t2.insertid(targname,linenum)
763           tm2 = tokmention(t2,myfile,linenum)
764           name = toknamestring(t2)
765           applytodict(linkhyperdict,targname,tm2)
766         else:
767           print "Error newcommand ommision in python code."
768           sys.exit(1)
769       elif rawname == "\\livetarg":
770         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
771       elif rawname == "\\livetargi":
772         tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
773       elif rawname == "\\livelink":
774         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
775       elif rawname == "\\livelinki":
776           tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
777       #elif rawname == "\\label":
778       #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
779       elif rawname == "\\refersec":
780         # does \ref
781         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
782       elif rawname == "\\referfol":
783         # does \vref from varioref package
784         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
785       elif rawname == "\\index":
786         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
787       elif rawname == "\\addtoindex":
788         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
789       elif rawname == "\\addtoindexx":
790         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
791       elif rawname == "\\addttindex":
792         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
793       elif rawname == "\\addttindexx":
794         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
795       elif rawname == "\\DWOPbregtwo":
796         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
797       elif rawname == "\\DWOPbregthree":
798         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
799       elif rawname == "\\DWOPbregfour":
800         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
801       elif rawname == "\\DWOPbregfive":
802         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
803       elif rawname == "\\DWOPbregeleven":
804         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
805       elif rawname == "\\MDfive":
806         tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
807       else:
808         fake = ""
809         # If missing anything important, perhaps turn ths on.
810         #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
811       tnumin = tnumin + tnumcount
812       continue
813     # Suffixes are LINK TARG INDX MARK NAME
814     commandname =rawnameiscommand(rawname,"LINK")
815     if len(commandname) > 0:
816       # index the DWname
817       # Link is to chap:DWname
818       if dwfnamecommsdict.has_key(commandname):
819         tm = tokmention(t,myfile,linenum)
820         linkname = makelinkname(commandname)
821         indxname = deloptionalprefix(commandname,"\\")
822         applytodict(linkhyperdict,linkname,tm)
823         applytodict(indexdict,indxname,tm);
824       else:
825         printodderr(rawname,commandname,myfile,linenum)
826       tnumin = tnumin + 1
827       continue
828     commandname =rawnameiscommand(rawname,"TARG")
829     if len(commandname) > 0:
830       # index DWname
831       # Set chap:DWname as having target defined
832       if dwfnamecommsdict.has_key(commandname):
833         tm = tokmention(t,myfile,linenum)
834         targname = makelinkname(commandname)
835         indxname = deloptionalprefix(commandname,"\\")
836         applytodict(targhyperdict,targname,tm)
837         applytodict(indexdict,indxname,tm);
838       else:
839         printodderr(rawname,commandname,myfile,linenum)
840       tnumin = tnumin + 1
841       continue
842     commandname =rawnameiscommand(rawname,"INDX")
843     if len(commandname) > 0:
844       # Index DWname
845       if dwfnamecommsdict.has_key(commandname):
846         tm = tokmention(t,myfile,linenum)
847         indexname = deloptionalprefix(commandname,"\\")
848         applytodict(indexdict,indexname,tm)
849       else:
850         printodderr(rawname,commandname,myfile,linenum)
851       tnumin = tnumin + 1
852       continue
853     commandname =rawnameiscommand(rawname,"MARK")
854     if len(commandname) > 0:
855       # set chap:DWname as target defined
856       # index DWname
857       if dwfnamecommsdict.has_key(commandname):
858         tm = tokmention(t,myfile,linenum)
859         targname = makelinkname(commandname)
860         applytodict(targhyperdict,targname,tm)
861         indexname = deloptionalprefix(commandname,"\\")
862         applytodict(indexdict,indexname,tm)
863       else:
864         printodderr(rawname,commandname,myfile,linenum)
865       tnumin = tnumin + 1
866       continue
867     commandname =rawnameiscommand(rawname,"NAME")
868     if len(commandname) > 0:
869       # No actions with NAME (but put in namedict anyway).
870       if dwfnamecommsdict.has_key(commandname):
871         tm = tokmention(t,myfile,linenum)
872         applytodict(namedict,commandname,tm)
873       else:
874         printodderr(rawname,commandname,myfile,linenum)
875       tnumin = tnumin + 1
876       continue
877     if rawname == "\\label":
878       # This is a builtin, not our newcommand.
879       tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
880       tnumin = tnumin + tnumcount
881       continue
882     if rawname == "\\hypertarget":
883       # This is a builtin, not our newcommand.
884       tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
885       tnumin = tnumin + tnumcount
886       continue
887     if rawname == "\\hyperlink":
888       # This is a builtin, not our newcommand.
889       tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
890       tnumin = tnumin + tnumcount
891       continue
892     # Some random data or text here.
893
894     tnumin = tnumin + 1
895     # We don't know what this is. Probably ok?
896     # End of for loop.
897   return linetoks
898
899 def sort_tokmlist(mylist):
900   aux = [ (''.join(x._token._tex),x) for x in mylist ]
901   aux.sort()
902   return[ (x[1]) for x in aux]
903
904 def printtokmention(ct,v):
905   n = v._token
906   name =  toknamestring(n)
907   f = v._file
908   l = v._line
909   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
910
911 # Are the lines close? If so 
912 def closetogether(l1,l2):
913   d = abs(l1 - l2)
914   if d < 4:
915     return "y"
916   return "n"
917
918 def shouldprintalldups(v):
919   if len(v) != 2:
920      return "y"
921   if v[0]._file != v[1]._file:
922      return "y"
923   if closetogether(v[0]._line,v[1]._line) == "y":
924         # Are the lines near one another? 
925         # If so a harmless duplication
926     return "n"
927   return "y"
928
929 def printtoomany(name,k,vlist):
930   print "Duplicate in %s: %s:" %(name,k)
931   ct = 0
932   for v in vlist:
933     printtokmention(ct,v)
934     ct = ct + 1
935
936 def checkduplicatetargs(dname,d):
937   targs = d.items()
938   for vi in targs:
939     (k,v) = vi
940     if len(v) > 1:
941       if shouldprintalldups(v) == "y":
942         printtoomany(dname,k,v)
943
944 def checkmissingtarg(name,targ,refs):
945   n = 0
946   rlist = refs.items()
947   for r in rlist:
948     (k,v) = r
949     if targ.has_key(k) == 0:
950        n = n + 1
951        print "Missing target from " + name + ":",k
952   if n == 0: print "No missing targets from " + name +"!"
953
954 def checkmissingref(name,targ,refs):
955   n = 0
956   rlist = targ.items()
957   for r in rlist:
958     (k,v) = r
959     if refs.has_key(k) == 0:
960        n = n + 1
961        print "Unused target from " + name + ":",k
962   if n == 0: print "No unused targets from " + name + "..."
963
964 def print_stats():
965   global dwfnamecommsdict
966   global newcommsdict
967   global targhyperdict
968   global linkhyperdict
969   global indexdict
970   global labeldict
971   global labelrefdict
972   
973   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
974   checkduplicatetargs("commands",newcommsdict)
975   checkduplicatetargs("hypertargets",targhyperdict)
976   checkduplicatetargs("labels",labeldict)
977
978   # Unused targets are nice to know
979   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
980   checkmissingref("labels",labeldict,labelrefdict)
981
982   # Missing targets are a problem to be FIXED
983   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
984   checkmissingtarg("labels",labeldict,labelrefdict)
985
986 # Perhaps these should be controlled by
987 # the command line.
988 debug   = "n"
989
990 def buildfilepaths(files,basetarg):
991   outlist = []
992   prefix = ""
993   for f in files:
994     prefix = ""
995     if len(basetarg) > 0:
996       prefix = basetarg
997     else:
998       prefix = os.path.join("..","latexdoc","")
999     outlist += [prefix + f]
1000   return outlist
1001 def read_all_args():
1002   filelist1 = []
1003   filelist2 = []
1004   baselist1 = []
1005   baselist2 = []
1006   basetarg = ""
1007   fileio.setkeepordeletecomments("d")
1008   if debug == "y":
1009     baselist1 = ["testrefer.tex"]
1010     baselist2 = ["testrefer.tex"]
1011     basetarg = "./"
1012   else:
1013     baselist1 = ["dwarfnamecmds.tex",
1014               "dwarf5.tex",
1015               "generaldescription.tex"]
1016
1017     baselist2 = ["dwarf5.tex",
1018               "attributesbytag.tex",
1019               "changesummary.tex",
1020               "compression.tex",
1021               "copyright.tex",
1022               "dataobject.tex",
1023               "datarepresentation.tex",
1024               "debugsectionrelationships.tex",
1025               "encodingdecoding.tex",
1026               "examples.tex",
1027               "foreword.tex",
1028               "generaldescription.tex",
1029               "gnulicense.tex",
1030               "introduction.tex",
1031               "otherdebugginginformation.tex",
1032               "programscope.tex",
1033               "sectionversionnumbers.tex",
1034               "splitobjects.tex",
1035               "typeentries.tex"]
1036   filelist1 = buildfilepaths(baselist1,basetarg)
1037   filelist2 = buildfilepaths(baselist2,basetarg)
1038
1039   if (len(filelist1) < 1) or (len(filelist2) < 1):
1040     print >> sys.stderr , "No files specified to refer.py, internal error."
1041     sys.exit(1)
1042   # Pickup all the newcommand instances.
1043   dwf = fileio.readFilelist(filelist1)
1044   dwf.dwtransformline(transfunc1)
1045
1046   # Now find all the uses.
1047   dwf2 = fileio.readFilelist(filelist2)
1048   dwf2.dwtransformline(transfunc2)
1049   print_stats()
1050
1051 if __name__ == '__main__':
1052   read_all_args()
1053