Add README.md
[dwarf-doc.git] / dwarf5 / tools / referb.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex(+) to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12 # This version turns each file into a single token list
13 # instead of line by line.
14 # As of June 14, 2014, it is not yet functioning.
15
16 # This is the simplest 'parse' of the .tex that we can manage
17 # while still finding what we want to find.
18 # One would hardly call it a parser, really.
19
20 # There are essentially three namespaces at present in the document.
21 # The hyperlink/hypertarget namespace.
22 # The label  vref ref namespace
23 #   which also involves  our refersec referfol.
24 # The index namespace, which we are not presently filling in very much. 
25
26 import sys
27 import os
28 import os.path
29 import fileio
30
31 # These two hold the commands we care about so we can
32 # bypass most lines easily in phase two.
33 # All the newdwfnamecommands
34 global dwfnamecommsdict
35 # All the newcommand instances.
36 global newcommsdict
37
38 # NAME suffix. No index, just text shows.
39 # Not very useful so far
40 global namedict
41
42
43 # Targets for \hypertarget and \hyperlink
44 #targhyperdict is all the \hyperarget instances.
45 #linkhyperdict is all the \hyperlink instances.
46 global targhyperdict
47 global linkhyperdict
48
49 # label targets (labels)
50 global labeldict
51 # \refersec \ref \vref to labels
52 global labelrefdict
53
54 # The index content for named things.
55 # The strings here are the links, the targets are
56 # built by the latex index software.
57 # So this dictionary is not really needed here (yet)
58 # and is not yet fully built up.
59 global indexdict
60
61
62 newcommsdict = {}
63 dwfnamecommsdict = {}
64
65 # Links meaning \livelink \livetarg \livetargi macros
66 #linksdict  = {}
67 #targsdict  = {}
68 namedict  = {}
69 targhyperdict= {}
70 linkhyperdict= {}
71 labeldict = {}
72 labelrefdict = {}
73 indexdict = {}
74
75 global linestoignore
76 linestoignore = []
77
78 # lines_to_ignore is a terrible hack.
79 def add_lines_to_ignore(myfile,lowline,highline):
80   global linestoignore
81   linestoignore += [(myfile._name,lowline,highline)]
82 def in_lines_to_ignore(myfile,line):
83   global linestoignore
84   n = myfile._name
85   for x in linestoignore:
86     (n,l,h) = x
87     if  myfile._name != n:
88       continue
89     if line < l:
90       continue
91     if line >h:
92       continue
93     return "y"
94   return "n"
95    
96
97 # a list of words to ignore: silly stuff.
98 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
99 "Standards":0,"Committee":0,"Version":0 }
100
101 class tokmention:
102   def __init__(self):
103     self._token = '' 
104     self._file = ""
105     self._line = 0
106   def __init__(self,tok,filename,line):
107     self._token = tok
108     self._file = filename
109     self._line = line
110
111 def ischar(tok,c):
112    if tok._class != "ind":
113       return "n"
114    if len(tok._tex) != 1:
115        return "n"
116    if tok._tex[0] != c:
117        return "n"
118    return "y"
119
120 def dwspace(tok):
121   if ischar(tok," ") == "y":
122     return "y"
123   if ischar(tok,"\t") == "y":
124     return "y"
125   return "n"
126   
127   
128 def isbrace(tok,brace):
129   if tok._class != "ind":
130      return "n"
131   if len(tok._tex) != 1:
132      return "n"
133   if brace == tok._tex[0]:
134      return "y"
135   return "n"
136
137 def toknamestring(t):
138   """ Turn a token into its string as a string """
139   return ''.join(t._tex)
140
141
142 def pickup(linetoks,tnumin,pattern,myfile,linenum,suppresserr):
143   """ The token pattern characters are
144   i meaning identifier
145   e meaning identifier, but ifnext token is }
146      we construct an empty identifier for it.
147   [space] meaning whitespace
148   { meaning left brace
149   } meaning right brace
150   * meaning any token except } and end-line
151   
152   Precondition:  linetoks[tnumin] is identifier (meaning a command)
153   Returns: a token list, one per non-space in the pattern.
154      For the *, the token is itself a list of whatever it contains.
155   """
156   outtoks = []
157   numabsorbed = 1
158   inlen = len(linetoks) 
159   curnum = tnumin
160   curtok = linetoks[curnum]
161   patterncharnum = -1
162   for c in pattern:
163     patterncharnum = patterncharnum + 1
164     if curnum >= inlen:
165       if suppresserr == "n":
166         print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
167       return outtoks,numabsorbed
168     curtok = linetoks[curnum]
169     if c == " ":
170       while dwspace(curtok) == "y":
171         curnum = curnum + 1
172         if curnum >= inlen:
173           if suppresserr == "n":
174             print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
175           return outtoks,numabsorbed
176         numabsorbed = numabsorbed + 1
177         curtok = linetoks[curnum]
178       continue
179     elif c == "i":
180       if curtok._class != "id":
181         if suppresserr == "n":
182           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
183         return outtoks,numabsorbed
184       numabsorbed = numabsorbed + 1
185       outtoks += [curtok]
186       curnum = curnum + 1
187       continue
188     elif c == "e":
189       if curtok._class != "id":
190         if isbrace(curtok,"}") == "y":
191           tk = fileio.dwtoken()
192           tk.insertid("")
193           outtoks += [tk]
194           # Do not update location.
195           continue
196         else:
197           if suppresserr == "n":
198             print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
199           return outtoks,numabsorbed
200       else: 
201         numabsorbed = numabsorbed + 1
202         outtoks += [curtok]
203         curnum = curnum + 1
204       continue
205     elif c == "{":
206       if isbrace(curtok,"{")  == "y":
207         outtoks += [curtok]
208         curnum = curnum + 1
209         numabsorbed = numabsorbed + 1
210       else:
211         if suppresserr == "n":
212           print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
213         return outtoks,numabsorbed
214     elif c == "}":
215       if isbrace(curtok,"}")  == "y":
216         outtoks += [curtok]
217         curnum = curnum + 1
218         numabsorbed = numabsorbed + 1
219       else:
220         if suppresserr == "n":
221           print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
222         return outtoks,numabsorbed
223     elif c == "[":
224       if isbrace(curtok,"[")  == "y":
225         outtoks += [curtok]
226         curnum = curnum + 1
227         numabsorbed = numabsorbed + 1
228       else:
229         if suppresserr == "n":
230           print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
231         return outtoks,numabsorbed
232     elif c == "]":
233       if isbrace(curtok,"]")  == "y":
234         outtoks += [curtok]
235         curnum = curnum + 1
236         numabsorbed = numabsorbed + 1
237       else:
238         if suppresserr == "n":
239           print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
240         return outtoks,numabsorbed
241     elif c == "*":
242       outlist = []
243       curtok = linetoks[curnum]
244       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
245         if dwspace(curtok) == "n":
246            outlist += [curtok]
247         curnum = curnum + 1
248         if curnum >= inlen:
249           outtoks += [outlist]
250           if patterncharnum < (len(pattern) -1): 
251             if suppresserr == "n":
252               print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
253           return outtoks,numabsorbed
254         numabsorbed = numabsorbed + 1
255         curtok = linetoks[curnum]
256       # Found a right brace, so done here.
257       outtoks += [outlist]
258     else:
259         if suppresserr == "n":
260           print "ERROR pattern had unexpected character ",pattern
261   return outtoks,numabsorbed
262
263 def printbadcommand(name,myfile,myline):
264   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
265
266 def applytodict(d,k,v):
267   keystring = k
268   if d.has_key(keystring) == 0:
269      d[keystring] =  [v]
270   else:
271      # This is a duplicate entry.
272      # We will report on it later as appropriate.
273      existing = d.get(keystring)
274      existing += [v]
275      d[keystring] =  existing
276
277 # See how many "{" there are on the line.
278 # return the count.
279 def countbraces(linetoks,tnumin):
280   lasttoknum = len(linetoks) -1
281   lb = 0
282   while tnumin < lasttoknum:
283     x = linetoks[tnumin] 
284     if x._class == "ind":
285        n = toknamestring(x)
286        if n == "{":
287           lb = lb + 1
288     tnumin = tnumin + 1
289   return lb
290      
291
292
293 # Here we try two different parses, the [] is optional
294 # with simplenametable.
295 def processbegin(linetoks,tnumin,myfile,linenum):
296   global targhyperdict
297   global indexdict
298   lbracecount = countbraces(linetoks,tnumin)
299   if lbracecount < 3:
300     return 1
301   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum,"y")
302   if len(ourtoks) == 13:
303     lcom = ourtoks[2]
304     lcomname = toknamestring(lcom)
305     if lcomname != "simplenametable":
306       return inlen
307     targ = ourtoks[11]
308     hypstr = toknamestring(targ)
309     hypmen = tokmention(targ,myfile,linenum)
310     applytodict(labeldict,hypstr,hypmen)
311     return inlen
312   ourtoks,inlen = pickup(linetoks,tnumin," i { i } { * } { i }",myfile,linenum,"y")
313   if len(ourtoks) < 10:
314       return inlen
315   lcom = ourtoks[2]
316   lcomname = toknamestring(lcom)
317   if lcomname != "simplenametable":
318       return inlen
319   targ = ourtoks[8]
320   hypstr = toknamestring(targ)
321   hypmen = tokmention(targ,myfile,linenum)
322   applytodict(labeldict,hypstr,hypmen)
323   return inlen
324
325 # If justlink == "y" this is a hypertarget command literally.
326 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
327   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
328   global targhyperdict
329   global indexdict
330   t = linetoks[tnumin]
331   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
332   if len(ourtoks) > 5:
333     t2 = ourtoks[2];
334     index = tokmention(t2,myfile,linenum)
335     name = toknamestring(t2)
336     applytodict(targhyperdict,name,index)
337     if justlink == "n":
338       t2 = ourtoks[5];
339       # Ignore for now.
340       #name = toknamestring(t2)
341       #if len(name) > 0:
342       #  index = tokmention(t2,myfile,linenum)
343       #  applytodict(indexdict,name,index)
344   else:
345     tn = toknamestring(linetoks[tnumin])
346     printbadcommand(tn,myfile,linenum)
347   return inlen
348 def livetargiprocess(linetoks,tnumin,myfile,linenum):
349   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
350   global targhyperdict
351   global indexdict
352   t = linetoks[tnumin]
353   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * } { * }",myfile,linenum,"n")
354   if len(ourtoks) > 5:
355     t2 = ourtoks[2];
356     index = tokmention(t2,myfile,linenum)
357     name = toknamestring(t2)
358     applytodict(targhyperdict,name,index)
359
360     # It is the final *, not the first, that is about the index.
361     #t2 = ourtoks[5];
362     #name = toknamestring(t2)
363     #if len(name) > 0:
364     #  index = tokmention(t2,myfile,linenum)
365     #  applytodict(indexdict,name,index)
366   else:
367     tn = toknamestring(linetoks[tnumin])
368     printbadcommand(tn,myfile,linenum)
369   return inlen
370 # if justlink == "y" it is a plain hyperlinkcommand
371 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
372   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
373   global linkhyperdict
374   global indexdict
375   t = linetoks[tnumin]
376   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
377   if len(ourtoks) > 5:
378     t2 = ourtoks[2];
379     index = tokmention(t2,myfile,linenum)
380     name = toknamestring(t2)
381     applytodict(linkhyperdict,name,index)
382
383     # can be multiword. For now do not bother with every index.
384     #t2 = ourtoks[5];
385     #index = tokmention(t2,myfile,linenum)
386     #name = toknamestring(t2)
387     #applytodict(indexdict,name,index)
388   else:
389     tn = toknamestring(linetoks[tnumin])
390     printbadcommand(tn,myfile,linenum)
391   return inlen
392 def labelprocess(linetoks,tnumin,myfile,linenum):
393   """ \label{alabel} """
394   global labeldict
395   t = linetoks[tnumin]
396   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
397   if len(ourtoks) > 2:
398     t2 = ourtoks[2];
399     index = tokmention(t2,myfile,linenum)
400     name = toknamestring(t2)
401     applytodict(labeldict,name,index)
402   else:
403     tn = toknamestring(linetoks[tnumin])
404     printbadcommand(tn,myfile,linenum)
405
406   return inlen
407 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
408   """ \addtoindex{strings} """
409   global indexdict
410   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
411   if len(ourtoks) > 2:
412     # The * means a list of tokens.
413     fake = ""
414     #t2 = ourtoks[2];
415     #index = tokmention(t2,myfile,linenum)
416     #name = toknamestring(t2)
417     #applytodict(indexdict,name,index)
418   else:
419     tn = toknamestring(linetoks[tnumin])
420     printbadcommand(tn,myfile,linenum)
421   return inlen
422 def hyperlinkname(name,tnumin,myfile,linenum):
423   global linkhyperdict
424   tkmod = fileio.dwtoken()
425   tkmod.insertid(name)
426   tm = tokmention(tkmod,myfile,linenum)
427   applytodict(linkhyperdict,name,tm)
428   return 1
429 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
430   """ \hyperlink{entryname} """
431   global linkhyperdict
432   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
433   if len(ourtoks) > 2:
434     t2 = ourtoks[2];
435     index = tokmention(t2,myfile,linenum)
436     name = toknamestring(t2)
437     applytodict(linkhyperdict,name,index)
438   else:
439     tn = toknamestring(linetoks[tnumin])
440     printbadcommand(tn,myfile,linenum)
441   return inlen
442
443 def indexprocess(linetoks,tnumin,myfile,linenum):
444   """ \index{indexentryname} """
445   global indexdict
446   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
447   if len(ourtoks) > 2:
448     fake = ""
449     # For now not bothering with index strings
450     #t2 = ourtoks[2];
451     #index = tokmention(t2,myfile,linenum)
452     #name = toknamestring(t2)
453     #applytodict(indexdict,name,index)
454   else:
455     tn = toknamestring(linetoks[tnumin])
456     printbadcommand(tn,myfile,linenum)
457   return inlen
458 def refersecprocess(linetoks,tnumin,myfile,linenum):
459   """ \refersec{label} """
460   global labelrefdict
461   t = linetoks[tnumin]
462   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
463   if len(ourtoks) > 2:
464     t2 = ourtoks[2];
465     index = tokmention(t2,myfile,linenum)
466     name = toknamestring(t2)
467     applytodict(labelrefdict,name,labelrefdict)
468   else:
469     tn = toknamestring(linetoks[tnumin])
470     printbadcommand(tn,myfile,linenum)
471   return inlen
472
473 def firstnonblank(linetoks):
474   tnum = 0
475   lasttoknum = len(linetoks)
476   while tnum < lasttoknum:
477     x = linetoks[tnum]
478     if x._class != "ind":
479       return tnum
480     if toknamestring(x) == " ":
481       tnum = tnum + 1
482       continue
483     elif toknamestring(x) == "\t":
484       tnum = tnum + 1
485       continue
486     return tnum 
487   return tnum
488 # Deals solely with finding new commands.
489 # This done as a first pass so we can recognize when tokens are
490 # really commands, something transfunc2, the second pass, 
491 # wants to know. 
492 def transfunc1(alltoks,myfile):
493   global dwfnamecommsdict
494   global newcommsdict
495
496   if len(alltoks) < 1:
497     return alltoks
498   tnum = firstnonblank(alltoks)
499   if tnum >= len(alltoks):
500     return linetoks
501   nexttoknum =tnum 
502   finaltoknum = int(len(alltoks))
503   while int(nexttoknum) < int(finaltoknum):
504     tnum = nexttoknum
505     initialtok = alltoks[tnum]
506     itokstring=toknamestring(initialtok)
507     if itokstring == "\\expandafter\\def\\csname":
508       nexttoknum =  int(nexttoknum) + 1
509       continue
510     if itokstring == "\\newcommand":
511       t1 = linetoks[tnum+1]
512       if not isbrace(t1,'{'):
513         print "Improper character in newcommand", myfile,initaltok._linenum
514         sys.exit(1)
515       t2 = linetoks[tnum+2]
516       if toknamestring(t2) == "\\simplenametablerule":
517          add_lines_to_ignore(myfile,linenum,linenum+18)
518       if toknamestring(t2) != "\\newdwfnamecommands":
519          tm = tokmention(t2,myfile,linenum)
520          applytodict(newcommsdict,toknamestring(t2),tm)
521       #Be silent on newdwfnamecommands, it is normal.
522       #else:
523       #   print "newcommand on newdwfnamecommands ignored intentionally."
524       nexttoknum = nexttoknum + 3;
525       continue
526     elif itokstring == "\\newdwfnamecommands":
527       t1 = linetoks[nexttoknum+1]
528       if not isbrace(t1,'{'):
529          print "Improper character in newdwfnamecommands", myfile._name,linenum
530          sys.exit(1)
531       # The token name string will be DWsomething and we want
532       # The token to appear as \DWsomething as that is how references
533       # The usages determine what secondary actions are applied.
534       # are coded.
535       t2 = linetoks[tnum+2]
536       tkmod = fileio.dwtoken()
537       tkmod.insertid("\\" + toknamestring(t2))
538       tm = tokmention(tkmod,myfile,linenum)
539       applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
540       nexttoknum = nexttoknum + 3;
541       continue
542     else:
543       nexttoknum =  int(nexttoknum) + 1
544   return alltoks
545
546
547 def delsuffix(n,suf):
548   slen = len(suf)
549   nlen = len(n)
550   lastcharnum = nlen - slen
551   outstring = n[0:lastcharnum]
552   return outstring
553 def deloptionalprefix(n,pref):
554   if not n.startswith(pref):
555     return n
556   plen = len(pref)
557   nlen = len(n)
558   outstring = n[plen:nlen]
559   return outstring
560
561 def printodderr(rawname,comname,myfile,linenum):
562   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
563
564 def rawnameiscommand(t,suff):
565    if not t.startswith("\\DW"):
566      return ""
567    if not t.endswith(suff):
568      return ""
569    commandname = delsuffix(t,suff)
570    return commandname
571
572 # Delete any leading backslash.
573 # Prefix the result with chap:
574 def makelinkname(t):
575    s = deloptionalprefix(t,"\\");
576    s2 = "chap:" + s
577    return s2;
578    
579
580 def gotonextline(alltoks,myfile,curtoknum,lasttoknum):
581   # Iterate till line number changes,
582   # return curtoknum + count, which
583   # might be to past lasttoknum.
584 FIXME
585
586 # Assumes all new commands known already.
587 # This deals with targets and links (various flavors).
588 def transfunc2(alltoks,myfile):
589   global newcommsdict
590   global dwfnamecommsdict
591   global newcommsdict
592
593   # Link naming target
594   global linkhyperdict
595   # TARG suffix
596   global targhyperdict
597   # INDX suffix
598   global indexdict
599   # NAME suffix
600   global namedict
601
602   if len(alltoks) < 1:
603     return alltoks
604   tnum = firstnonblank(alltoks)
605   if tnum >= len(alltoks):
606     return linetoks
607   nexttoknum =tnum
608   lasttoknum = int(len(alltoks))
609   initialtok = alltoks[tnum]
610   currentlineno = initialtok._linenum
611   while int(nexttoknum) < int(lasttoknum):
612     tnumin = nexttoknum
613     initialtok = alltoks[tnumin]
614     itokstring=toknamestring(initialtok)
615
616     # Skip all the newcommand stuff.
617     if itokstring == "\\newcommand":
618       nexttoknum = gotonextline(alltoks, myfile,nexttoknum,lasttoknum)
619       continue
620     elif itokstring == '\\newdwfnamecommands':
621       nexttoknum = gotonextline(alltoks, myfile,nexttoknum,lasttoknum)
622       continue
623   
624 FIXME
625     if int(tnumin) > int(lasttoknum):
626         break
627     t = linetoks[tnumin]
628     if t._class != "id":
629         tnumin = tnumin + 1
630         continue
631     rawname = toknamestring(t)
632     commandname=""
633     #rawnameiscommand(rawname,"",basecommand)
634     if rawname == "\\expandafter\\def\\csname":
635         return linetoks
636     if rawname == "\\begin":
637         tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
638         tnumin = tnumin + tnumcount
639         continue
640     if dwfnamecommsdict.has_key(rawname):
641         # We know this one. 
642         # It is a default case name reference.
643         # index the DWname
644         # Link is to chap:DWname
645         tm = tokmention(t,myfile,linenum)
646         linkname = makelinkname(rawname)
647         indxname = deloptionalprefix(commandname,"\\")
648         applytodict(indexdict,indxname,tm);
649   
650         applytodict(linkhyperdict,linkname,tm);
651         tnumin = tnumin + 1
652         continue
653       if  newcommsdict.has_key(rawname):
654         # We know this one. We have to see what it is
655         # To decide what to do.
656         # some DWOPbreg*  DWOPreg*   and MDfive are special.
657         # A variety of other such defined commands are irrelevant to us here.
658   
659         tnumcount = 1
660         if rawname == "\\livetarg":
661           tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
662         elif rawname == "\\livetargi":
663           tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
664         elif rawname == "\\livelink":
665           tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
666         elif rawname == "\\livelinki":
667           tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
668         #elif rawname == "\\label":
669         #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
670         elif rawname == "\\refersec":
671           # does \ref
672           tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
673         elif rawname == "\\referfol":
674           # does \vref from varioref package
675           tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
676         elif rawname == "\\index":
677           tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
678         elif rawname == "\\addtoindex":
679           tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
680         elif rawname == "\\addtoindexx":
681           tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
682         elif rawname == "\\addttindex":
683           tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
684         elif rawname == "\\addttindexx":
685           tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
686         elif rawname == "\\DWOPbregtwo":
687           tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
688         elif rawname == "\\DWOPbregthree":
689           tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
690         elif rawname == "\\DWOPbregfour":
691           tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
692         elif rawname == "\\DWOPbregfive":
693           tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
694         elif rawname == "\\DWOPbregeleven":
695           tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
696         elif rawname == "\\MDfive":
697           tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
698         else:
699           fake = ""
700           # If missing anything important, perhaps turn ths on.
701           #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
702         tnumin = tnumin + tnumcount
703         continue
704       # Suffixes are LINK TARG INDX MARK NAME
705       commandname =rawnameiscommand(rawname,"LINK")
706       if len(commandname) > 0:
707         # index the DWname
708         # Link is to chap:DWname
709         if dwfnamecommsdict.has_key(commandname):
710           tm = tokmention(t,myfile,linenum)
711           linkname = makelinkname(commandname)
712           indxname = deloptionalprefix(commandname,"\\")
713           applytodict(linkhyperdict,linkname,tm)
714           applytodict(indexdict,indxname,tm);
715         else:
716           printodderr(rawname,commandname,myfile,linenum)
717         tnumin = tnumin + 1
718         continue
719     commandname =rawnameiscommand(rawname,"TARG")
720     if len(commandname) > 0:
721         # index DWname
722         # Set chap:DWname as having target defined
723         if dwfnamecommsdict.has_key(commandname):
724           tm = tokmention(t,myfile,linenum)
725           targname = makelinkname(commandname)
726           indxname = deloptionalprefix(commandname,"\\")
727           applytodict(targhyperdict,targname,tm)
728           applytodict(indexdict,indxname,tm);
729         else:
730           printodderr(rawname,commandname,myfile,linenum)
731         tnumin = tnumin + 1
732         continue
733     commandname =rawnameiscommand(rawname,"INDX")
734     if len(commandname) > 0:
735         # Index DWname
736         if dwfnamecommsdict.has_key(commandname):
737           tm = tokmention(t,myfile,linenum)
738           indexname = deloptionalprefix(commandname,"\\")
739           applytodict(indexdict,indexname,tm)
740         else:
741           printodderr(rawname,commandname,myfile,linenum)
742         tnumin = tnumin + 1
743         continue
744     commandname =rawnameiscommand(rawname,"MARK")
745     if len(commandname) > 0:
746         # set chap:DWname as target defined
747         # index DWname
748         if dwfnamecommsdict.has_key(commandname):
749           tm = tokmention(t,myfile,linenum)
750           targname = makelinkname(commandname)
751           applytodict(targhyperdict,targname,tm)
752           indexname = deloptionalprefix(commandname,"\\")
753           applytodict(indexdict,indexname,tm)
754         else:
755           printodderr(rawname,commandname,myfile,linenum)
756         tnumin = tnumin + 1
757         continue
758     commandname =rawnameiscommand(rawname,"NAME")
759     if len(commandname) > 0:
760         # No actions with NAME (but put in namedict anyway).
761         if dwfnamecommsdict.has_key(commandname):
762           tm = tokmention(t,myfile,linenum)
763           applytodict(namedict,commandname,tm)
764         else:
765           printodderr(rawname,commandname,myfile,linenum)
766         tnumin = tnumin + 1
767         continue
768       if rawname == "\\label":
769         # This is a builtin, not our newcommand.
770         tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
771         tnumin = tnumin + tnumcount
772         continue
773       if rawname == "\\hypertarget":
774         # This is a builtin, not our newcommand.
775         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
776         tnumin = tnumin + tnumcount
777         continue
778       if rawname == "\\hyperlink":
779         # This is a builtin, not our newcommand.
780         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
781         tnumin = tnumin + tnumcount
782         continue
783       # Some random data or text here.
784   
785       nexttoknum = nexttoknum + 1
786     # We don't know what this is. Probably ok?
787     # End of for loop on all tokens in the file.
788     nexttokum = nexttoknum + 1
789   # end of FOR loop.
790   return alltoks
791   
792 def sort_tokmlist(mylist):
793   aux = [ (''.join(x._token._tex),x) for x in mylist ]
794   aux.sort()
795   return[ (x[1]) for x in aux]
796
797 def printtokmention(ct,v):
798   n = v._token
799   name =  toknamestring(n)
800   f = v._file
801   l = v._line
802   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
803
804 # Are the lines close? If so 
805 def closetogether(l1,l2):
806   d = abs(l1 - l2)
807   if d < 4:
808     return "y"
809   return "n"
810
811 def shouldprintalldups(v):
812   if len(v) != 2:
813      return "y"
814   if v[0]._file != v[1]._file:
815      return "y"
816   if closetogether(v[0]._line,v[1]._line) == "y":
817         # Are the lines near one another? 
818         # If so a harmless duplication
819     return "n"
820   return "y"
821
822 def printtoomany(name,k,vlist):
823   print "Duplicate in %s: %s:" %(name,k)
824   ct = 0
825   for v in vlist:
826     printtokmention(ct,v)
827     ct = ct + 1
828
829 def checkduplicatetargs(dname,d):
830   targs = d.items()
831   for vi in targs:
832     (k,v) = vi
833     if len(v) > 1:
834       if shouldprintalldups(v) == "y":
835         printtoomany(dname,k,v)
836
837 def checkmissingtarg(name,targ,refs):
838   n = 0
839   rlist = refs.items()
840   for r in rlist:
841     (k,v) = r
842     if targ.has_key(k) == 0:
843        n = n + 1
844        print "Missing target from " + name + ":",k
845   if n == 0: print "No missing targets from " + name +"!"
846
847 def checkmissingref(name,targ,refs):
848   n = 0
849   rlist = targ.items()
850   for r in rlist:
851     (k,v) = r
852     if refs.has_key(k) == 0:
853        n = n + 1
854        print "Unused target from " + name + ":",k
855   if n == 0: print "No unused targets from " + name + "..."
856
857 def print_stats():
858   global dwfnamecommsdict
859   global newcommsdict
860   global targhyperdict
861   global linkhyperdict
862   global indexdict
863   global labeldict
864   global labelrefdict
865   
866   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
867   checkduplicatetargs("commands",newcommsdict)
868   checkduplicatetargs("hypertargets",targhyperdict)
869   checkduplicatetargs("labels",labeldict)
870
871   # Unused targets are nice to know
872   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
873   checkmissingref("labels",labeldict,labelrefdict)
874
875   # Missing targets are a problem to be FIXED
876   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
877   checkmissingtarg("labels",labeldict,labelrefdict)
878
879 # Perhaps these should be controlled by
880 # the command line.
881 debug   = "n"
882
883 def buildfilepaths(files,basetarg):
884   outlist = []
885   prefix = ""
886   for f in files:
887     prefix = ""
888     if len(basetarg) > 0:
889       prefix = basetarg
890     else:
891       prefix = os.path.join("..","latexdoc","")
892     outlist += [prefix + f]
893   return outlist
894 def read_all_args():
895   filelist1 = []
896   filelist2 = []
897   baselist1 = []
898   baselist2 = []
899   basetarg = ""
900   fileio.setkeepordeletecomments("d")
901   if debug == "y":
902     baselist1 = ["testrefer.tex"]
903     baselist2 = ["testrefer.tex"]
904     basetarg = "./"
905   else:
906     baselist1 = ["dwarfnamecmds.tex",
907               "dwarf5.tex",
908               "generaldescription.tex"]
909
910     baselist2 = ["dwarf5.tex",
911               "attributesbytag.tex",
912               "changesummary.tex",
913               "compression.tex",
914               "copyright.tex",
915               "dataobject.tex",
916               "datarepresentation.tex",
917               "debugsectionrelationships.tex",
918               "encodingdecoding.tex",
919               "examples.tex",
920               "foreword.tex",
921               "generaldescription.tex",
922               "gnulicense.tex",
923               "introduction.tex",
924               "otherdebugginginformation.tex",
925               "programscope.tex",
926               "sectionversionnumbers.tex",
927               "splitobjects.tex",
928               "typeentries.tex"]
929   filelist1 = buildfilepaths(baselist1,basetarg)
930   filelist2 = buildfilepaths(baselist2,basetarg)
931
932   if (len(filelist1) < 1) or (len(filelist2) < 1):
933     print >> sys.stderr , "No files specified to refer.py, internal error."
934     sys.exit(1)
935   # Pickup all the newcommand instances.
936   dwf = fileio.readFilelist(filelist1)
937   dwf.dwtransformtoks(transfunc1)
938
939   # Now find all the uses.
940   dwf2 = fileio.readFilelist(filelist2)
941   dwf2.dwtransformtoks(transfunc2)
942   print_stats()
943
944 if __name__ == '__main__':
945   read_all_args()
946