tools/refer.py: Now seems to work finding problems.
[dwarf-doc.git] / dwarf5 / tools / refer.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex(+) to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12 # This is the simplest 'parse' of the .tex that we can manage
13 # while still finding what we want to find.
14 # One would hardly call it a parser, really.
15
16 # There are essentially three namespaces at present in the document.
17 # The hyperlink/hypertarget namespace.
18 # The label  vref ref namespace
19 #   which also involves  our refersec referfol.
20 # The index namespace, which we are not presently filling in very much. 
21
22 import sys
23 import fileio
24
25 # These two hold the commands we care about so we can
26 # bypass most lines easily in phase two.
27 # All the newdwfnamecommands
28 global dwfnamecommsdict
29 # All the newcommand instances.
30 global newcommsdict
31
32 # NAME suffix. No index, just text shows.
33 # Not very useful so far
34 global namedict
35
36
37 # Targets for \hypertarget and \hyperlink
38 #targhyperdict is all the \hyperarget instances.
39 #linkhyperdict is all the \hyperlink instances.
40 global targhyperdict
41 global linkhyperdict
42
43 # label targets (labels)
44 global labeldict
45 # \refersec \ref \vref to labels
46 global labelrefdict
47
48 # The index content for named things.
49 # The strings here are the links, the targets are
50 # built by the latex index software.
51 # So this dictionary is not really needed here (yet)
52 # and is not yet fully built up.
53 global indexdict
54
55
56 newcommsdict = {}
57 dwfnamecommsdict = {}
58
59 # Links meaning \livelink \livetarg \livetargi macros
60 #linksdict  = {}
61 #targsdict  = {}
62 namedict  = {}
63 targhyperdict= {}
64 linkhyperdict= {}
65 labeldict = {}
66 labelrefdict = {}
67 indexdict = {}
68
69 global linestoignore
70 linestoignore = []
71
72 # lines_to_ignore is a terrible hack.
73 def add_lines_to_ignore(myfile,lowline,highline):
74   global linestoignore
75   linestoignore += [(myfile._name,lowline,highline)]
76 def in_lines_to_ignore(myfile,line):
77   global linestoignore
78   n = myfile._name
79   for x in linestoignore:
80     (n,l,h) = x
81     if  myfile._name != n:
82       continue
83     if line < l:
84       continue
85     if line >h:
86       continue
87     return "y"
88   return "n"
89    
90
91 # a list of words to ignore: silly stuff.
92 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
93 "Standards":0,"Committee":0,"Version":0 }
94
95 class tokmention:
96   def __init__(self):
97     self._token = '' 
98     self._file = ""
99     self._line = 0
100   def __init__(self,tok,filename,line):
101     self._token = tok
102     self._file = filename
103     self._line = line
104
105 def ischar(tok,c):
106    if tok._class != "ind":
107       return "n"
108    if len(tok._tex) != 1:
109        return "n"
110    if tok._tex[0] != c:
111        return "n"
112    return "y"
113
114 def dwspace(tok):
115   if ischar(tok," ") == "y":
116     return "y"
117   if ischar(tok,"\t") == "y":
118     return "y"
119   return "n"
120   
121   
122 def isbrace(tok,brace):
123   if tok._class != "ind":
124      return "n"
125   if len(tok._tex) != 1:
126      return "n"
127   if brace == tok._tex[0]:
128      return "y"
129   return "n"
130
131 def toknamestring(t):
132   """ Turn a token into its string as a string """
133   return ''.join(t._tex)
134
135
136 def pickup(linetoks,tnumin,pattern,myfile,linenum,suppresserr):
137   """ The token pattern characters are
138   i meaning identifier
139   e meaning identifier, but ifnext token is }
140      we construct an empty identifier for it.
141   [space] meaning whitespace
142   { meaning left brace
143   } meaning right brace
144   * meaning any token except } and end-line
145   
146   Precondition:  linetoks[tnumin] is identifier (meaning a command)
147   Returns: a token list, one per non-space in the pattern.
148      For the *, the token is itself a list of whatever it contains.
149   """
150   outtoks = []
151   numabsorbed = 1
152   inlen = len(linetoks) 
153   curnum = tnumin
154   curtok = linetoks[curnum]
155   patterncharnum = -1
156   for c in pattern:
157     patterncharnum = patterncharnum + 1
158     if curnum >= inlen:
159       if suppresserr == "n":
160         print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
161       return outtoks,numabsorbed
162     curtok = linetoks[curnum]
163     if c == " ":
164       while dwspace(curtok) == "y":
165         curnum = curnum + 1
166         if curnum >= inlen:
167           if suppresserr == "n":
168             print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
169           return outtoks,numabsorbed
170         numabsorbed = numabsorbed + 1
171         curtok = linetoks[curnum]
172       continue
173     elif c == "i":
174       if curtok._class != "id":
175         if suppresserr == "n":
176           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
177         return outtoks,numabsorbed
178       numabsorbed = numabsorbed + 1
179       outtoks += [curtok]
180       curnum = curnum + 1
181       continue
182     elif c == "e":
183       if curtok._class != "id":
184         if isbrace(curtok,"}") == "y":
185           tk = fileio.dwtoken()
186           tk.insertid("")
187           outtoks += [tk]
188           # Do not update location.
189           continue
190         else:
191           if suppresserr == "n":
192             print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
193           return outtoks,numabsorbed
194       else: 
195         numabsorbed = numabsorbed + 1
196         outtoks += [curtok]
197         curnum = curnum + 1
198       continue
199     elif c == "{":
200       if isbrace(curtok,"{")  == "y":
201         outtoks += [curtok]
202         curnum = curnum + 1
203         numabsorbed = numabsorbed + 1
204       else:
205         if suppresserr == "n":
206           print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
207         return outtoks,numabsorbed
208     elif c == "}":
209       if isbrace(curtok,"}")  == "y":
210         outtoks += [curtok]
211         curnum = curnum + 1
212         numabsorbed = numabsorbed + 1
213       else:
214         if suppresserr == "n":
215           print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
216         return outtoks,numabsorbed
217     elif c == "[":
218       if isbrace(curtok,"[")  == "y":
219         outtoks += [curtok]
220         curnum = curnum + 1
221         numabsorbed = numabsorbed + 1
222       else:
223         if suppresserr == "n":
224           print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
225         return outtoks,numabsorbed
226     elif c == "]":
227       if isbrace(curtok,"]")  == "y":
228         outtoks += [curtok]
229         curnum = curnum + 1
230         numabsorbed = numabsorbed + 1
231       else:
232         if suppresserr == "n":
233           print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
234         return outtoks,numabsorbed
235     elif c == "*":
236       outlist = []
237       curtok = linetoks[curnum]
238       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
239         if dwspace(curtok) == "n":
240            outlist += [curtok]
241         curnum = curnum + 1
242         if curnum >= inlen:
243           outtoks += [outlist]
244           if patterncharnum < (len(pattern) -1): 
245             if suppresserr == "n":
246               print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
247           return outtoks,numabsorbed
248         numabsorbed = numabsorbed + 1
249         curtok = linetoks[curnum]
250       # Found a right brace, so done here.
251       outtoks += [outlist]
252     else:
253         if suppresserr == "n":
254           print "ERROR pattern had unexpected character ",pattern
255   return outtoks,numabsorbed
256
257 def printbadcommand(name,myfile,myline):
258   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
259
260 def applytodict(d,k,v):
261   keystring = k
262   if d.has_key(keystring) == 0:
263      d[keystring] =  [v]
264   else:
265      # This is a duplicate entry.
266      # We will report on it later as appropriate.
267      existing = d.get(keystring)
268      existing += [v]
269      d[keystring] =  existing
270
271 # See how many "{" there are on the line.
272 # return the count.
273 def countbraces(linetoks,tnumin):
274   lasttoknum = len(linetoks) -1
275   lb = 0
276   while tnumin < lasttoknum:
277     x = linetoks[tnumin] 
278     if x._class == "ind":
279        n = toknamestring(x)
280        if n == "{":
281           lb = lb + 1
282     tnumin = tnumin + 1
283   return lb
284      
285
286
287 # Here we try two different parses, the [] is optional
288 # with simplenametable.
289 def processbegin(linetoks,tnumin,myfile,linenum):
290   global targhyperdict
291   global indexdict
292   lbracecount = countbraces(linetoks,tnumin)
293   if lbracecount < 3:
294     return 1
295   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum,"y")
296   if len(ourtoks) == 13:
297     lcom = ourtoks[2]
298     lcomname = toknamestring(lcom)
299     if lcomname != "simplenametable":
300       return inlen
301     targ = ourtoks[11]
302     hypstr = toknamestring(targ)
303     hypmen = tokmention(targ,myfile,linenum)
304     applytodict(labeldict,hypstr,hypmen)
305     return inlen
306   ourtoks,inlen = pickup(linetoks,tnumin," i { i } { * } { i }",myfile,linenum,"y")
307   if len(ourtoks) < 10:
308       return inlen
309   lcom = ourtoks[2]
310   lcomname = toknamestring(lcom)
311   if lcomname != "simplenametable":
312       return inlen
313   targ = ourtoks[8]
314   hypstr = toknamestring(targ)
315   hypmen = tokmention(targ,myfile,linenum)
316   applytodict(labeldict,hypstr,hypmen)
317   return inlen
318
319 # If justlink == "y" this is a hypertarget command literally.
320 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
321   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
322   global targhyperdict
323   global indexdict
324   t = linetoks[tnumin]
325   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
326   if len(ourtoks) > 5:
327     t2 = ourtoks[2];
328     index = tokmention(t2,myfile,linenum)
329     name = toknamestring(t2)
330     applytodict(targhyperdict,name,index)
331     if justlink == "n":
332       t2 = ourtoks[5];
333       # Ignore for now.
334       #name = toknamestring(t2)
335       #if len(name) > 0:
336       #  index = tokmention(t2,myfile,linenum)
337       #  applytodict(indexdict,name,index)
338   else:
339     tn = toknamestring(linetoks[tnumin])
340     printbadcommand(tn,myfile,linenum)
341   return inlen
342 def livetargiprocess(linetoks,tnumin,myfile,linenum):
343   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
344   global targhyperdict
345   global indexdict
346   t = linetoks[tnumin]
347   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { e } { * }",myfile,linenum,"n")
348   if len(ourtoks) > 5:
349     t2 = ourtoks[2];
350     index = tokmention(t2,myfile,linenum)
351     name = toknamestring(t2)
352     applytodict(targhyperdict,name,index)
353
354     t2 = ourtoks[5];
355     name = toknamestring(t2)
356     if len(name) > 0:
357       index = tokmention(t2,myfile,linenum)
358       applytodict(indexdict,name,index)
359   else:
360     tn = toknamestring(linetoks[tnumin])
361     printbadcommand(tn,myfile,linenum)
362   return inlen
363 # if justlink == "y" it is a plain hyperlinkcommand
364 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
365   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
366   global linkhyperdict
367   global indexdict
368   t = linetoks[tnumin]
369   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
370   if len(ourtoks) > 5:
371     t2 = ourtoks[2];
372     index = tokmention(t2,myfile,linenum)
373     name = toknamestring(t2)
374     applytodict(linkhyperdict,name,index)
375
376     # can be multiword. For now do not bother with every index.
377     #t2 = ourtoks[5];
378     #index = tokmention(t2,myfile,linenum)
379     #name = toknamestring(t2)
380     #applytodict(indexdict,name,index)
381   else:
382     tn = toknamestring(linetoks[tnumin])
383     printbadcommand(tn,myfile,linenum)
384   return inlen
385 def labelprocess(linetoks,tnumin,myfile,linenum):
386   """ \label{alabel} """
387   global labeldict
388   t = linetoks[tnumin]
389   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
390   if len(ourtoks) > 2:
391     t2 = ourtoks[2];
392     index = tokmention(t2,myfile,linenum)
393     name = toknamestring(t2)
394     applytodict(labeldict,name,index)
395   else:
396     tn = toknamestring(linetoks[tnumin])
397     printbadcommand(tn,myfile,linenum)
398
399   return inlen
400 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
401   """ \addtoindex{strings} """
402   global indexdict
403   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
404   if len(ourtoks) > 2:
405     # The * means a list of tokens.
406     fake = ""
407     #t2 = ourtoks[2];
408     #index = tokmention(t2,myfile,linenum)
409     #name = toknamestring(t2)
410     #applytodict(indexdict,name,index)
411   else:
412     tn = toknamestring(linetoks[tnumin])
413     printbadcommand(tn,myfile,linenum)
414   return inlen
415 def hyperlinkname(name,tnumin,myfile,linenum):
416   global linkhyperdict
417   tkmod = fileio.dwtoken()
418   tkmod.insertid(name)
419   tm = tokmention(tkmod,myfile,linenum)
420   applytodict(linkhyperdict,name,tm)
421   return 1
422 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
423   """ \hyperlink{entryname} """
424   global linkhyperdict
425   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
426   if len(ourtoks) > 2:
427     t2 = ourtoks[2];
428     index = tokmention(t2,myfile,linenum)
429     name = toknamestring(t2)
430     applytodict(linkhyperdict,name,index)
431   else:
432     tn = toknamestring(linetoks[tnumin])
433     printbadcommand(tn,myfile,linenum)
434   return inlen
435
436 def indexprocess(linetoks,tnumin,myfile,linenum):
437   """ \index{indexentryname} """
438   global indexdict
439   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
440   if len(ourtoks) > 2:
441     fake = ""
442     # For now not bothering with index strings
443     #t2 = ourtoks[2];
444     #index = tokmention(t2,myfile,linenum)
445     #name = toknamestring(t2)
446     #applytodict(indexdict,name,index)
447   else:
448     tn = toknamestring(linetoks[tnumin])
449     printbadcommand(tn,myfile,linenum)
450   return inlen
451 def refersecprocess(linetoks,tnumin,myfile,linenum):
452   """ \refersec{label} """
453   global labelrefdict
454   t = linetoks[tnumin]
455   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
456   if len(ourtoks) > 2:
457     t2 = ourtoks[2];
458     index = tokmention(t2,myfile,linenum)
459     name = toknamestring(t2)
460     applytodict(labelrefdict,name,labelrefdict)
461   else:
462     tn = toknamestring(linetoks[tnumin])
463     printbadcommand(tn,myfile,linenum)
464   return inlen
465
466 def firstnonblank(linetoks):
467   tnum = 0
468   lasttoknum = len(linetoks)
469   while tnum < lasttoknum:
470     x = linetoks[tnum]
471     if x._class != "ind":
472       return tnum
473     if toknamestring(x) == " ":
474       tnum = tnum + 1
475       continue
476     elif toknamestring(x) == "\t":
477       tnum = tnum + 1
478       continue
479     return tnum 
480   return tnum
481 # Deals solely with finding new commands.
482 # This done as a first pass so we can recognize when tokens are
483 # really commands, something transfunc2, the second pass, 
484 # wants to know. 
485 def transfunc1(linetoks,myfile,linenum):
486   global dwfnamecommsdict
487   global newcommsdict
488
489   if len(linetoks) < 1:
490     return linetoks
491   tnum = firstnonblank(linetoks)
492   if tnum >= len(linetoks):
493     return linetoks
494   initialtok = linetoks[tnum]
495   itokstring=toknamestring(initialtok)
496   if itokstring == "\\expandafter\\def\\csname":
497     return linetoks
498   if in_lines_to_ignore(myfile,linenum) == "y":
499     return linetoks
500   if itokstring == "\\newcommand":
501     t1 = linetoks[tnum+1]
502     if not isbrace(t1,'{'):
503       print "Improper character in newcommand", myfile,linenum
504       sys.exit(1)
505     t2 = linetoks[tnum+2]
506     if toknamestring(t2) == "\\simplenametablerule":
507        add_lines_to_ignore(myfile,linenum,linenum+18)
508     if toknamestring(t2) != "\\newdwfnamecommands":
509        tm = tokmention(t2,myfile,linenum)
510        applytodict(newcommsdict,toknamestring(t2),tm)
511     #Be silent on newdwfnamecommands, it is normal.
512     #else:
513     #   print "newcommand on newdwfnamecommands ignored intentionally."
514     return linetoks
515   elif itokstring == "\\newdwfnamecommands":
516     t1 = linetoks[tnum+1]
517     if not isbrace(t1,'{'):
518        print "Improper character in newdwfnamecommands", myfile._name,linenum
519        sys.exit(1)
520     # The token name string will be DWsomething and we want
521     # The token to appear as \DWsomething as that is how references
522     # The usages determine what secondary actions are applied.
523     # are coded.
524     t2 = linetoks[tnum+2]
525     tkmod = fileio.dwtoken()
526     tkmod.insertid("\\" + toknamestring(t2))
527     tm = tokmention(tkmod,myfile,linenum)
528     applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
529     return linetoks
530   return linetoks
531
532
533 def delsuffix(n,suf):
534   slen = len(suf)
535   nlen = len(n)
536   lastcharnum = nlen - slen
537   outstring = n[0:lastcharnum]
538   return outstring
539 def deloptionalprefix(n,pref):
540   if not n.startswith(pref):
541     return n
542   plen = len(pref)
543   nlen = len(n)
544   outstring = n[plen:nlen]
545   return outstring
546
547 def printodderr(rawname,comname,myfile,linenum):
548   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
549
550 def rawnameiscommand(t,suff):
551    if not t.startswith("\\DW"):
552      return ""
553    if not t.endswith(suff):
554      return ""
555    commandname = delsuffix(t,suff)
556    return commandname
557
558 # Delete any leading backslash.
559 # Prefix the result with chap:
560 def makelinkname(t):
561    s = deloptionalprefix(t,"\\");
562    s2 = "chap:" + s
563    return s2;
564    
565
566 # Assumes all new commands known already.
567 # This deals with targets and links (various flavors).
568 def transfunc2(linetoks,myfile,linenum):
569   global newcommsdict
570   global dwfnamecommsdict
571   global newcommsdict
572
573   # Link naming target
574   global linkhyperdict
575   # TARG suffix
576   global targhyperdict
577   # INDX suffix
578   global indexdict
579   # NAME suffix
580   global namedict
581
582   if len(linetoks) < 1:
583     return linetoks
584   if in_lines_to_ignore(myfile,linenum) == "y":
585     return linetoks
586   initialtok = linetoks[0]
587   itokstring=toknamestring(initialtok)
588   # Skip all the newcommand stuff.
589   if itokstring == "\\newcommand":
590     return linetoks
591   elif itokstring == '\\newdwfnamecommands':
592     return linetoks
593
594   # Now deal with a regular line.
595
596   tnumin = 0
597   changes = 0
598   lasttoknum = len(linetoks) -1
599   for x in linetoks:
600     if int(tnumin) > int(lasttoknum):
601       break
602     t = linetoks[tnumin]
603     if t._class != "id":
604       tnumin = tnumin + 1
605       continue
606     rawname = toknamestring(t)
607     commandname=""
608     #rawnameiscommand(rawname,"",basecommand)
609     if rawname == "\\expandafter\\def\\csname":
610       return linetoks
611     if rawname == "\\begin":
612       tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
613       tnumin = tnumin + tnumcount
614       continue
615     if dwfnamecommsdict.has_key(rawname):
616       # We know this one. 
617       # It is a default case name reference.
618       # index the DWname
619       # Link is to chap:DWname
620       tm = tokmention(t,myfile,linenum)
621       linkname = makelinkname(rawname)
622       indxname = deloptionalprefix(commandname,"\\")
623       applytodict(indexdict,indxname,tm);
624
625       applytodict(linkhyperdict,linkname,tm);
626       tnumin = tnumin + 1
627       continue
628     if  newcommsdict.has_key(rawname):
629       # We know this one. We have to see what it is
630       # To decide what to do.
631       # some DWOPbreg*  DWOPreg*   and MDfive are special.
632       # A variety of other such defined commands are irrelevant to us here.
633
634       tnumcount = 1
635       if rawname == "\\livetarg":
636         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
637       elif rawname == "\\livetargi":
638         tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
639       elif rawname == "\\livelink":
640         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
641       elif rawname == "\\livelinki":
642         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
643       #elif rawname == "\\label":
644       #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
645       elif rawname == "\\refersec":
646         # does \ref
647         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
648       elif rawname == "\\referfol":
649         # does \vref from varioref package
650         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
651       elif rawname == "\\index":
652         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
653       elif rawname == "\\addtoindex":
654         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
655       elif rawname == "\\addtoindexx":
656         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
657       elif rawname == "\\addttindex":
658         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
659       elif rawname == "\\addttindexx":
660         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
661       elif rawname == "\\DWOPbregtwo":
662         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
663       elif rawname == "\\DWOPbregthree":
664         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
665       elif rawname == "\\DWOPbregfour":
666         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
667       elif rawname == "\\DWOPbregfive":
668         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
669       elif rawname == "\\DWOPbregeleven":
670         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
671       elif rawname == "\\MDfive":
672         tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
673       else:
674         fake = ""
675         # If missing anything important, perhaps turn ths on.
676         #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
677       tnumin = tnumin + tnumcount
678       continue
679     # Suffixes are LINK TARG INDX MARK NAME
680     commandname =rawnameiscommand(rawname,"LINK")
681     if len(commandname) > 0:
682       # index the DWname
683       # Link is to chap:DWname
684       if dwfnamecommsdict.has_key(commandname):
685         tm = tokmention(t,myfile,linenum)
686         linkname = makelinkname(commandname)
687         indxname = deloptionalprefix(commandname,"\\")
688         applytodict(linkhyperdict,linkname,tm)
689         applytodict(indexdict,indxname,tm);
690       else:
691         printodderr(rawname,commandname,myfile,linenum)
692       tnumin = tnumin + 1
693       continue
694     commandname =rawnameiscommand(rawname,"TARG")
695     if len(commandname) > 0:
696       # index DWname
697       # Set chap:DWname as having target defined
698       if dwfnamecommsdict.has_key(commandname):
699         tm = tokmention(t,myfile,linenum)
700         targname = makelinkname(commandname)
701         indxname = deloptionalprefix(commandname,"\\")
702         applytodict(targhyperdict,targname,tm)
703         applytodict(indexdict,indxname,tm);
704       else:
705         printodderr(rawname,commandname,myfile,linenum)
706       tnumin = tnumin + 1
707       continue
708     commandname =rawnameiscommand(rawname,"INDX")
709     if len(commandname) > 0:
710       # Index DWname
711       if dwfnamecommsdict.has_key(commandname):
712         tm = tokmention(t,myfile,linenum)
713         indexname = deloptionalprefix(commandname,"\\")
714         applytodict(indexdict,indexname,tm)
715       else:
716         printodderr(rawname,commandname,myfile,linenum)
717       tnumin = tnumin + 1
718       continue
719     commandname =rawnameiscommand(rawname,"MARK")
720     if len(commandname) > 0:
721       # set chap:DWname as target defined
722       # index DWname
723       if dwfnamecommsdict.has_key(commandname):
724         tm = tokmention(t,myfile,linenum)
725         applytodict(targhyperdict,commandname,tm)
726         indexname = deloptionalprefix(commandname,"\\")
727         applytodict(indexdict,indexname,tm)
728       else:
729         printodderr(rawname,commandname,myfile,linenum)
730       tnumin = tnumin + 1
731       continue
732     commandname =rawnameiscommand(rawname,"NAME")
733     if len(commandname) > 0:
734       # No actions with NAME (but put in namedict anyway).
735       if dwfnamecommsdict.has_key(commandname):
736         tm = tokmention(t,myfile,linenum)
737         applytodict(namedict,commandname,tm)
738       else:
739         printodderr(rawname,commandname,myfile,linenum)
740       tnumin = tnumin + 1
741       continue
742     if rawname == "\\label":
743       # This is a builtin, not our newcommand.
744       tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
745       tnumin = tnumin + tnumcount
746       continue
747     if rawname == "\\hypertarget":
748       # This is a builtin, not our newcommand.
749       tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
750       tnumin = tnumin + tnumcount
751       continue
752     if rawname == "\\hyperlink":
753       # This is a builtin, not our newcommand.
754       tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
755       tnumin = tnumin + tnumcount
756       continue
757     # Some random data or text here.
758
759     tnumin = tnumin + 1
760     # We don't know what this is. Probably ok?
761     # End of for loop.
762   return linetoks
763
764 def sort_tokmlist(mylist):
765   aux = [ (''.join(x._token._tex),x) for x in mylist ]
766   aux.sort()
767   return[ (x[1]) for x in aux]
768
769 def printtokmention(ct,v):
770   n = v._token
771   name =  toknamestring(n)
772   f = v._file
773   l = v._line
774   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
775
776 # Are the lines close? If so 
777 def closetogether(l1,l2):
778   d = abs(l1 - l2)
779   if d < 4:
780     return "y"
781   return "n"
782
783 def shouldprintalldups(v):
784   if len(v) != 2:
785      return "y"
786   if v[0]._file != v[1]._file:
787      return "y"
788   if closetogether(v[0]._line,v[1]._line) == "y":
789         # Are the lines near one another? 
790         # If so a harmless duplication
791     return "n"
792   return "y"
793
794 def printtoomany(name,k,vlist):
795   print "Duplicate in %s: %s:" %(name,k)
796   ct = 0
797   for v in vlist:
798     printtokmention(ct,v)
799     ct = ct + 1
800
801 def checkduplicatetargs(dname,d):
802   targs = d.items()
803   for vi in targs:
804     (k,v) = vi
805     if len(v) > 1:
806       if shouldprintalldups(v) == "y":
807         printtoomany(dname,k,v)
808
809 def checkmissingtarg(name,targ,refs):
810   rlist = refs.items()
811   for r in rlist:
812     (k,v) = r
813     if targ.has_key(k) == 0:
814        print "target missing from",name,":",k
815
816 def checkmissingref(name,targ,refs):
817   rlist = targ.items()
818   for r in rlist:
819     (k,v) = r
820     if refs.has_key(k) == 0:
821        print "Unused target from",name,":",k
822   
823 def print_stats():
824   global dwfnamecommsdict
825   global newcommsdict
826   global targhyperdict
827   global linkhyperdict
828   global indexdict
829   global labeldict
830   global labelrefdict
831   
832   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
833   checkduplicatetargs("commands",newcommsdict)
834   checkduplicatetargs("hypertargets",targhyperdict)
835   checkduplicatetargs("labels",labeldict)
836
837   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
838   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
839
840   checkmissingtarg("labels",labeldict,labelrefdict)
841   checkmissingref("labels",labeldict,labelrefdict)
842
843 # Perhaps these should be controlled by
844 # the command line.
845 debug   = "n"
846 winpath = "n"
847 def buildfilepaths(files,basetarg):
848   outlist = []
849   prefix = ""
850   for f in files:
851     prefix = ""
852     if len(basetarg) > 0:
853       prefix = basetarg
854     elif winpath == "y":
855       prefix = "..\\latexdoc\\"
856     else:
857       prefix = "../latexdoc/"
858     outlist += [prefix + f]
859   return outlist
860 def read_all_args():
861   filelist1 = []
862   filelist2 = []
863   baselist1 = []
864   baselist2 = []
865   basetarg = ""
866   fileio.setkeepordeletecomments("d")
867   if debug == "y":
868     baselist1 = ["testrefer.tex"]
869     baselist2 = ["testrefer.tex"]
870     basetarg = "./"
871   else:
872     baselist1 = ["dwarfnamecmds.tex",
873               "dwarf5.tex",
874               "generaldescription.tex"]
875
876     baselist2 = ["dwarf5.tex",
877               "attributesbytag.tex",
878               "changesummary.tex",
879               "compression.tex",
880               "copyright.tex",
881               "dataobject.tex",
882               "datarepresentation.tex",
883               "debugsectionrelationships.tex",
884               "encodingdecoding.tex",
885               "examples.tex",
886               "foreword.tex",
887               "generaldescription.tex",
888               "gnulicense.tex",
889               "introduction.tex",
890               "otherdebugginginformation.tex",
891               "programscope.tex",
892               "sectionversionnumbers.tex",
893               "splitobjects.tex",
894               "typeentries.tex"]
895   filelist1 = buildfilepaths(baselist1,basetarg)
896   filelist2 = buildfilepaths(baselist2,basetarg)
897
898   if (len(filelist1) < 1) or (len(filelist2) < 1):
899     print >> sys.stderr , "No files specified to refer.py, internal error."
900     sys.exit(1)
901   # Pickup all the newcommand instances.
902   dwf = fileio.readFilelist(filelist1)
903   dwf.dwtransformline(transfunc1)
904
905   # Now find all the uses.
906   dwf2 = fileio.readFilelist(filelist2)
907   dwf2.dwtransformline(transfunc2)
908   print_stats()
909
910 if __name__ == '__main__':
911   read_all_args()
912