tools/fileio.py was not tokenizing
[dwarf-doc.git] / dwarf5 / tools / refer.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex(+) to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12 # This version assumes relevant commands are used
13 # all-on-one-line, not with operands on successive lines.
14
15 # This is the simplest 'parse' of the .tex that we can manage
16 # while still finding what we want to find.
17 # One would hardly call it a parser, really.
18
19 # There are essentially three namespaces at present in the document.
20 # The hyperlink/hypertarget namespace.
21 # The label  vref ref namespace
22 #   which also involves  our refersec referfol.
23 # The index namespace, which we are not presently filling in very much. 
24
25 import sys
26 import os
27 import os.path
28 import fileio
29
30 # These two hold the commands we care about so we can
31 # bypass most lines easily in phase two.
32 # All the newdwfnamecommands
33 global dwfnamecommsdict
34 # All the newcommand instances.
35 global newcommsdict
36
37 # NAME suffix. No index, just text shows.
38 # Not very useful so far
39 global namedict
40
41
42 # Targets for \hypertarget and \hyperlink
43 #targhyperdict is all the \hyperarget instances.
44 #linkhyperdict is all the \hyperlink instances.
45 global targhyperdict
46 global linkhyperdict
47
48 # label targets (labels)
49 global labeldict
50 # \refersec \ref \vref to labels
51 global labelrefdict
52
53 # The index content for named things.
54 # The strings here are the links, the targets are
55 # built by the latex index software.
56 # So this dictionary is not really needed here (yet)
57 # and is not yet fully built up.
58 global indexdict
59
60
61 newcommsdict = {}
62 dwfnamecommsdict = {}
63
64 # Links meaning \livelink \livetarg \livetargi macros
65 #linksdict  = {}
66 #targsdict  = {}
67 namedict  = {}
68 targhyperdict= {}
69 linkhyperdict= {}
70 labeldict = {}
71 labelrefdict = {}
72 indexdict = {}
73
74 global linestoignore
75 linestoignore = []
76
77 # lines_to_ignore is a terrible hack.
78 def add_lines_to_ignore(myfile,lowline,highline):
79   global linestoignore
80   linestoignore += [(myfile._name,lowline,highline)]
81 def in_lines_to_ignore(myfile,line):
82   global linestoignore
83   n = myfile._name
84   for x in linestoignore:
85     (n,l,h) = x
86     if  myfile._name != n:
87       continue
88     if line < l:
89       continue
90     if line >h:
91       continue
92     return "y"
93   return "n"
94    
95
96 # a list of words to ignore: silly stuff.
97 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
98 "Standards":0,"Committee":0,"Version":0 }
99
100 class tokmention:
101   def __init__(self):
102     self._token = '' 
103     self._file = ""
104     self._line = 0
105   def __init__(self,tok,filename,line):
106     self._token = tok
107     self._file = filename
108     self._line = line
109
110 def ischar(tok,c):
111    if tok._class != "ind":
112       return "n"
113    if len(tok._tex) != 1:
114        return "n"
115    if tok._tex[0] != c:
116        return "n"
117    return "y"
118
119 def dwspace(tok):
120   if ischar(tok," ") == "y":
121     return "y"
122   if ischar(tok,"\t") == "y":
123     return "y"
124   return "n"
125   
126   
127 def isbrace(tok,brace):
128   if tok._class != "ind":
129      return "n"
130   if len(tok._tex) != 1:
131      return "n"
132   if brace == tok._tex[0]:
133      return "y"
134   return "n"
135
136 def toknamestring(t):
137   """ Turn a token into its string as a string """
138   return ''.join(t._tex)
139
140
141 def pickup(linetoks,tnumin,pattern,myfile,linenum,suppresserr):
142   """ The token pattern characters are
143   i meaning identifier
144   e meaning identifier, but ifnext token is }
145      we construct an empty identifier for it.
146   [space] meaning whitespace
147   { meaning left brace
148   } meaning right brace
149   * meaning any token except } and end-line
150   
151   Precondition:  linetoks[tnumin] is identifier (meaning a command)
152   Returns: a token list, one per non-space in the pattern.
153      For the *, the token is itself a list of whatever it contains.
154   """
155   outtoks = []
156   numabsorbed = 1
157   inlen = len(linetoks) 
158   curnum = tnumin
159   curtok = linetoks[curnum]
160   patterncharnum = -1
161   for c in pattern:
162     patterncharnum = patterncharnum + 1
163     if curnum >= inlen:
164       if suppresserr == "n":
165         print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
166       return outtoks,numabsorbed
167     curtok = linetoks[curnum]
168     if c == " ":
169       while dwspace(curtok) == "y":
170         curnum = curnum + 1
171         if curnum >= inlen:
172           if suppresserr == "n":
173             print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
174           return outtoks,numabsorbed
175         numabsorbed = numabsorbed + 1
176         curtok = linetoks[curnum]
177       continue
178     elif c == "i":
179       if curtok._class != "id":
180         if suppresserr == "n":
181           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
182         return outtoks,numabsorbed
183       numabsorbed = numabsorbed + 1
184       outtoks += [curtok]
185       curnum = curnum + 1
186       continue
187     elif c == "e":
188       if curtok._class != "id":
189         if isbrace(curtok,"}") == "y":
190           tk = fileio.dwtoken()
191           tk.insertid("",linenum)
192           outtoks += [tk]
193           # Do not update location.
194           continue
195         else:
196           if suppresserr == "n":
197             print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
198           return outtoks,numabsorbed
199       else: 
200         numabsorbed = numabsorbed + 1
201         outtoks += [curtok]
202         curnum = curnum + 1
203       continue
204     elif c == "{":
205       if isbrace(curtok,"{")  == "y":
206         outtoks += [curtok]
207         curnum = curnum + 1
208         numabsorbed = numabsorbed + 1
209       else:
210         if suppresserr == "n":
211           print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
212         return outtoks,numabsorbed
213     elif c == "}":
214       if isbrace(curtok,"}")  == "y":
215         outtoks += [curtok]
216         curnum = curnum + 1
217         numabsorbed = numabsorbed + 1
218       else:
219         if suppresserr == "n":
220           print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
221         return outtoks,numabsorbed
222     elif c == "[":
223       if isbrace(curtok,"[")  == "y":
224         outtoks += [curtok]
225         curnum = curnum + 1
226         numabsorbed = numabsorbed + 1
227       else:
228         if suppresserr == "n":
229           print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
230         return outtoks,numabsorbed
231     elif c == "]":
232       if isbrace(curtok,"]")  == "y":
233         outtoks += [curtok]
234         curnum = curnum + 1
235         numabsorbed = numabsorbed + 1
236       else:
237         if suppresserr == "n":
238           print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
239         return outtoks,numabsorbed
240     elif c == "*":
241       outlist = []
242       curtok = linetoks[curnum]
243       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
244         if dwspace(curtok) == "n":
245            outlist += [curtok]
246         curnum = curnum + 1
247         if curnum >= inlen:
248           outtoks += [outlist]
249           if patterncharnum < (len(pattern) -1): 
250             if suppresserr == "n":
251               print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
252           return outtoks,numabsorbed
253         numabsorbed = numabsorbed + 1
254         curtok = linetoks[curnum]
255       # Found a right brace, so done here.
256       outtoks += [outlist]
257     else:
258         if suppresserr == "n":
259           print "ERROR pattern had unexpected character ",pattern
260   return outtoks,numabsorbed
261
262 def printbadcommand(name,myfile,myline):
263   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
264
265 def applytodict(d,k,v):
266   keystring = k
267   if d.has_key(keystring) == 0:
268      d[keystring] =  [v]
269   else:
270      # This is a duplicate entry.
271      # We will report on it later as appropriate.
272      existing = d.get(keystring)
273      existing += [v]
274      d[keystring] =  existing
275
276 # See how many "{" there are on the line.
277 # return the count.
278 def countbraces(linetoks,tnumin):
279   lasttoknum = len(linetoks) -1
280   lb = 0
281   while tnumin < lasttoknum:
282     x = linetoks[tnumin] 
283     if x._class == "ind":
284        n = toknamestring(x)
285        if n == "{":
286           lb = lb + 1
287     tnumin = tnumin + 1
288   return lb
289      
290
291
292 # Here we try two different parses, the [] is optional
293 # with simplenametable.
294 def processbegin(linetoks,tnumin,myfile,linenum):
295   global targhyperdict
296   global indexdict
297   lbracecount = countbraces(linetoks,tnumin)
298   if lbracecount < 3:
299     return 1
300   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum,"y")
301   if len(ourtoks) == 13:
302     lcom = ourtoks[2]
303     lcomname = toknamestring(lcom)
304     if lcomname != "simplenametable":
305       return inlen
306     targ = ourtoks[11]
307     hypstr = toknamestring(targ)
308     hypmen = tokmention(targ,myfile,linenum)
309     applytodict(labeldict,hypstr,hypmen)
310     return inlen
311   ourtoks,inlen = pickup(linetoks,tnumin," i { i } { * } { i }",myfile,linenum,"y")
312   if len(ourtoks) < 10:
313       return inlen
314   lcom = ourtoks[2]
315   lcomname = toknamestring(lcom)
316   if lcomname != "simplenametable":
317       return inlen
318   targ = ourtoks[8]
319   hypstr = toknamestring(targ)
320   hypmen = tokmention(targ,myfile,linenum)
321   applytodict(labeldict,hypstr,hypmen)
322   return inlen
323
324 # If justlink == "y" this is a hypertarget command literally.
325 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
326   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
327   global targhyperdict
328   global indexdict
329   t = linetoks[tnumin]
330   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
331   if len(ourtoks) > 5:
332     t2 = ourtoks[2];
333     index = tokmention(t2,myfile,linenum)
334     name = toknamestring(t2)
335     applytodict(targhyperdict,name,index)
336     if justlink == "n":
337       t2 = ourtoks[5];
338       # Ignore for now.
339       #name = toknamestring(t2)
340       #if len(name) > 0:
341       #  index = tokmention(t2,myfile,linenum)
342       #  applytodict(indexdict,name,index)
343   else:
344     tn = toknamestring(linetoks[tnumin])
345     printbadcommand(tn,myfile,linenum)
346   return inlen
347 def livetargiprocess(linetoks,tnumin,myfile,linenum):
348   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
349   global targhyperdict
350   global indexdict
351   t = linetoks[tnumin]
352   # The first * here used to be e but that was insufficiently general.
353   # The index is generated from the entire  * (token list 8) , not from the
354   # initial  *, token list 5
355   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * } { * }",myfile,linenum,"n")
356   if len(ourtoks) > 5:
357     t2 = ourtoks[2];
358     index = tokmention(t2,myfile,linenum)
359     name = toknamestring(t2)
360     applytodict(targhyperdict,name,index)
361
362     # WARNING: looking at 5 this way was wrong.
363     #t2 = ourtoks[5];
364     #name = toknamestring(t2)
365     #if len(name) > 0:
366     #  index = tokmention(t2,myfile,linenum)
367     #  applytodict(indexdict,name,index)
368   else:
369     tn = toknamestring(linetoks[tnumin])
370     printbadcommand(tn,myfile,linenum)
371   return inlen
372 # if justlink == "y" it is a plain hyperlinkcommand
373 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
374   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
375   global linkhyperdict
376   global indexdict
377   t = linetoks[tnumin]
378   if justlink == "y":
379     ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
380     if len(ourtoks) > 5:
381       t2 = ourtoks[2];
382       index = tokmention(t2,myfile,linenum)
383       name = toknamestring(t2)
384       applytodict(linkhyperdict,name,index)
385     else:
386       tn = toknamestring(linetoks[tnumin])
387       printbadcommand(tn,myfile,linenum)
388   else:
389     ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
390     if len(ourtoks) == 4:
391       t2 = ourtoks[2];
392       index = tokmention(t2,myfile,linenum)
393       name = toknamestring(t2)
394       applytodict(linkhyperdict,name,index)
395     else:
396       tn = toknamestring(linetoks[tnumin])
397       printbadcommand(tn,myfile,linenum)
398   return inlen
399 def labelprocess(linetoks,tnumin,myfile,linenum):
400   """ \label{alabel} """
401   global labeldict
402   t = linetoks[tnumin]
403   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
404   if len(ourtoks) > 2:
405     t2 = ourtoks[2];
406     index = tokmention(t2,myfile,linenum)
407     name = toknamestring(t2)
408     applytodict(labeldict,name,index)
409   else:
410     tn = toknamestring(linetoks[tnumin])
411     printbadcommand(tn,myfile,linenum)
412
413   return inlen
414 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
415   """ \addtoindex{strings} """
416   global indexdict
417   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
418   if len(ourtoks) > 2:
419     # The * means a list of tokens.
420     fake = ""
421     #t2 = ourtoks[2];
422     #index = tokmention(t2,myfile,linenum)
423     #name = toknamestring(t2)
424     #applytodict(indexdict,name,index)
425   else:
426     tn = toknamestring(linetoks[tnumin])
427     printbadcommand(tn,myfile,linenum)
428   return inlen
429 def hyperlinkname(name,tnumin,myfile,linenum):
430   global linkhyperdict
431   tkmod = fileio.dwtoken()
432   tkmod.insertid(name,linenum)
433   tm = tokmention(tkmod,myfile,linenum)
434   applytodict(linkhyperdict,name,tm)
435   return 1
436 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
437   """ \hyperlink{entryname} """
438   global linkhyperdict
439   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
440   if len(ourtoks) > 2:
441     t2 = ourtoks[2];
442     index = tokmention(t2,myfile,linenum)
443     name = toknamestring(t2)
444     applytodict(linkhyperdict,name,index)
445   else:
446     tn = toknamestring(linetoks[tnumin])
447     printbadcommand(tn,myfile,linenum)
448   return inlen
449
450 def indexprocess(linetoks,tnumin,myfile,linenum):
451   """ \index{indexentryname} """
452   global indexdict
453   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
454   if len(ourtoks) > 2:
455     fake = ""
456     # For now not bothering with index strings
457     #t2 = ourtoks[2];
458     #index = tokmention(t2,myfile,linenum)
459     #name = toknamestring(t2)
460     #applytodict(indexdict,name,index)
461   else:
462     tn = toknamestring(linetoks[tnumin])
463     printbadcommand(tn,myfile,linenum)
464   return inlen
465 def refersecprocess(linetoks,tnumin,myfile,linenum):
466   """ \refersec{label} """
467   global labelrefdict
468   t = linetoks[tnumin]
469   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
470   if len(ourtoks) > 2:
471     t2 = ourtoks[2];
472     index = tokmention(t2,myfile,linenum)
473     name = toknamestring(t2)
474     applytodict(labelrefdict,name,labelrefdict)
475   else:
476     tn = toknamestring(linetoks[tnumin])
477     printbadcommand(tn,myfile,linenum)
478   return inlen
479
480 def firstnonblank(linetoks):
481   tnum = 0
482   lasttoknum = len(linetoks)
483   while tnum < lasttoknum:
484     x = linetoks[tnum]
485     if x._class != "ind":
486       return tnum
487     if toknamestring(x) == " ":
488       tnum = tnum + 1
489       continue
490     elif toknamestring(x) == "\t":
491       tnum = tnum + 1
492       continue
493     return tnum 
494   return tnum
495 # Deals solely with finding new commands.
496 # This done as a first pass so we can recognize when tokens are
497 # really commands, something transfunc2, the second pass, 
498 # wants to know. 
499 def transfunc1(linetoks,myfile,linenum):
500   global dwfnamecommsdict
501   global newcommsdict
502
503   if len(linetoks) < 1:
504     return linetoks
505   tnum = firstnonblank(linetoks)
506   if tnum >= len(linetoks):
507     return linetoks
508   initialtok = linetoks[tnum]
509   itokstring=toknamestring(initialtok)
510   if itokstring == "\\expandafter\\def\\csname":
511     return linetoks
512   if in_lines_to_ignore(myfile,linenum) == "y":
513     return linetoks
514   if itokstring == "\\newcommand":
515     t1 = linetoks[tnum+1]
516     if not isbrace(t1,'{'):
517       print "Improper character in newcommand", myfile,linenum
518       sys.exit(1)
519     t2 = linetoks[tnum+2]
520     if toknamestring(t2) == "\\simplenametablerule":
521        add_lines_to_ignore(myfile,linenum,linenum+18)
522     if toknamestring(t2) != "\\newdwfnamecommands":
523        tm = tokmention(t2,myfile,linenum)
524        applytodict(newcommsdict,toknamestring(t2),tm)
525     #Be silent on newdwfnamecommands, it is normal.
526     #else:
527     #   print "newcommand on newdwfnamecommands ignored intentionally."
528     return linetoks
529   elif itokstring == "\\newdwfnamecommands":
530     t1 = linetoks[tnum+1]
531     if not isbrace(t1,'{'):
532        print "Improper character in newdwfnamecommands", myfile._name,linenum
533        sys.exit(1)
534     # The token name string will be DWsomething and we want
535     # The token to appear as \DWsomething as that is how references
536     # The usages determine what secondary actions are applied.
537     # are coded.
538     t2 = linetoks[tnum+2]
539     tkmod = fileio.dwtoken()
540     tkmod.insertid("\\" + toknamestring(t2),linenum)
541     tm = tokmention(tkmod,myfile,linenum)
542     applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
543     return linetoks
544   return linetoks
545
546
547 def delsuffix(n,suf):
548   slen = len(suf)
549   nlen = len(n)
550   lastcharnum = nlen - slen
551   outstring = n[0:lastcharnum]
552   return outstring
553 def deloptionalprefix(n,pref):
554   if not n.startswith(pref):
555     return n
556   plen = len(pref)
557   nlen = len(n)
558   outstring = n[plen:nlen]
559   return outstring
560
561 def printodderr(rawname,comname,myfile,linenum):
562   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
563
564 def rawnameiscommand(t,suff):
565    if not t.startswith("\\DW"):
566      return ""
567    if not t.endswith(suff):
568      return ""
569    commandname = delsuffix(t,suff)
570    return commandname
571
572 # Delete any leading backslash.
573 # Prefix the result with chap:
574 def makelinkname(t):
575    s = deloptionalprefix(t,"\\");
576    s2 = "chap:" + s
577    return s2;
578    
579
580 # Assumes all new commands known already.
581 # This deals with targets and links (various flavors).
582 def transfunc2(linetoks,myfile,linenum):
583   global newcommsdict
584   global dwfnamecommsdict
585   global newcommsdict
586
587   # Link naming target
588   global linkhyperdict
589   # TARG suffix
590   global targhyperdict
591   # INDX suffix
592   global indexdict
593   # NAME suffix
594   global namedict
595
596   if len(linetoks) < 1:
597     return linetoks
598   if in_lines_to_ignore(myfile,linenum) == "y":
599     return linetoks
600   initialtok = linetoks[0]
601   itokstring=toknamestring(initialtok)
602   # Skip all the newcommand stuff.
603   if itokstring == "\\newcommand":
604     return linetoks
605   elif itokstring == '\\newdwfnamecommands':
606     return linetoks
607
608   # Now deal with a regular line.
609
610   tnumin = 0
611   changes = 0
612   lasttoknum = len(linetoks) -1
613   for x in linetoks:
614     if int(tnumin) > int(lasttoknum):
615       break
616     t = linetoks[tnumin]
617     if t._class != "id":
618       tnumin = tnumin + 1
619       continue
620     rawname = toknamestring(t)
621     commandname=""
622     #rawnameiscommand(rawname,"",basecommand)
623     if rawname == "\\expandafter\\def\\csname":
624       return linetoks
625     if rawname == "\\begin":
626       tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
627       tnumin = tnumin + tnumcount
628       continue
629     if dwfnamecommsdict.has_key(rawname):
630       # We know this one. 
631       # It is a default case name reference.
632       # index the DWname
633       # Link is to chap:DWname
634       tm = tokmention(t,myfile,linenum)
635       linkname = makelinkname(rawname)
636       indxname = deloptionalprefix(commandname,"\\")
637       applytodict(indexdict,indxname,tm);
638
639       applytodict(linkhyperdict,linkname,tm);
640       tnumin = tnumin + 1
641       continue
642     if  newcommsdict.has_key(rawname):
643       # We know this one. We have to see what it is
644       # To decide what to do.
645       # some DWOPbreg*  DWOPreg*   and MDfive are special.
646       # A variety of other such defined commands are irrelevant to us here.
647
648       tnumcount = 1
649       if rawname == "\\livetarg":
650         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
651       elif rawname == "\\livetargi":
652         tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
653       elif rawname == "\\livelink":
654         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
655       elif rawname == "\\livelinki":
656         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
657       #elif rawname == "\\label":
658       #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
659       elif rawname == "\\refersec":
660         # does \ref
661         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
662       elif rawname == "\\referfol":
663         # does \vref from varioref package
664         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
665       elif rawname == "\\index":
666         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
667       elif rawname == "\\addtoindex":
668         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
669       elif rawname == "\\addtoindexx":
670         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
671       elif rawname == "\\addttindex":
672         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
673       elif rawname == "\\addttindexx":
674         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
675       elif rawname == "\\DWOPbregtwo":
676         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
677       elif rawname == "\\DWOPbregthree":
678         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
679       elif rawname == "\\DWOPbregfour":
680         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
681       elif rawname == "\\DWOPbregfive":
682         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
683       elif rawname == "\\DWOPbregeleven":
684         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
685       elif rawname == "\\MDfive":
686         tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
687       else:
688         fake = ""
689         # If missing anything important, perhaps turn ths on.
690         #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
691       tnumin = tnumin + tnumcount
692       continue
693     # Suffixes are LINK TARG INDX MARK NAME
694     commandname =rawnameiscommand(rawname,"LINK")
695     if len(commandname) > 0:
696       # index the DWname
697       # Link is to chap:DWname
698       if dwfnamecommsdict.has_key(commandname):
699         tm = tokmention(t,myfile,linenum)
700         linkname = makelinkname(commandname)
701         indxname = deloptionalprefix(commandname,"\\")
702         applytodict(linkhyperdict,linkname,tm)
703         applytodict(indexdict,indxname,tm);
704       else:
705         printodderr(rawname,commandname,myfile,linenum)
706       tnumin = tnumin + 1
707       continue
708     commandname =rawnameiscommand(rawname,"TARG")
709     if len(commandname) > 0:
710       # index DWname
711       # Set chap:DWname as having target defined
712       if dwfnamecommsdict.has_key(commandname):
713         tm = tokmention(t,myfile,linenum)
714         targname = makelinkname(commandname)
715         indxname = deloptionalprefix(commandname,"\\")
716         applytodict(targhyperdict,targname,tm)
717         applytodict(indexdict,indxname,tm);
718       else:
719         printodderr(rawname,commandname,myfile,linenum)
720       tnumin = tnumin + 1
721       continue
722     commandname =rawnameiscommand(rawname,"INDX")
723     if len(commandname) > 0:
724       # Index DWname
725       if dwfnamecommsdict.has_key(commandname):
726         tm = tokmention(t,myfile,linenum)
727         indexname = deloptionalprefix(commandname,"\\")
728         applytodict(indexdict,indexname,tm)
729       else:
730         printodderr(rawname,commandname,myfile,linenum)
731       tnumin = tnumin + 1
732       continue
733     commandname =rawnameiscommand(rawname,"MARK")
734     if len(commandname) > 0:
735       # set chap:DWname as target defined
736       # index DWname
737       if dwfnamecommsdict.has_key(commandname):
738         tm = tokmention(t,myfile,linenum)
739         targname = makelinkname(commandname)
740         applytodict(targhyperdict,targname,tm)
741         indexname = deloptionalprefix(commandname,"\\")
742         applytodict(indexdict,indexname,tm)
743       else:
744         printodderr(rawname,commandname,myfile,linenum)
745       tnumin = tnumin + 1
746       continue
747     commandname =rawnameiscommand(rawname,"NAME")
748     if len(commandname) > 0:
749       # No actions with NAME (but put in namedict anyway).
750       if dwfnamecommsdict.has_key(commandname):
751         tm = tokmention(t,myfile,linenum)
752         applytodict(namedict,commandname,tm)
753       else:
754         printodderr(rawname,commandname,myfile,linenum)
755       tnumin = tnumin + 1
756       continue
757     if rawname == "\\label":
758       # This is a builtin, not our newcommand.
759       tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
760       tnumin = tnumin + tnumcount
761       continue
762     if rawname == "\\hypertarget":
763       # This is a builtin, not our newcommand.
764       tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
765       tnumin = tnumin + tnumcount
766       continue
767     if rawname == "\\hyperlink":
768       # This is a builtin, not our newcommand.
769       tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
770       tnumin = tnumin + tnumcount
771       continue
772     # Some random data or text here.
773
774     tnumin = tnumin + 1
775     # We don't know what this is. Probably ok?
776     # End of for loop.
777   return linetoks
778
779 def sort_tokmlist(mylist):
780   aux = [ (''.join(x._token._tex),x) for x in mylist ]
781   aux.sort()
782   return[ (x[1]) for x in aux]
783
784 def printtokmention(ct,v):
785   n = v._token
786   name =  toknamestring(n)
787   f = v._file
788   l = v._line
789   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
790
791 # Are the lines close? If so 
792 def closetogether(l1,l2):
793   d = abs(l1 - l2)
794   if d < 4:
795     return "y"
796   return "n"
797
798 def shouldprintalldups(v):
799   if len(v) != 2:
800      return "y"
801   if v[0]._file != v[1]._file:
802      return "y"
803   if closetogether(v[0]._line,v[1]._line) == "y":
804         # Are the lines near one another? 
805         # If so a harmless duplication
806     return "n"
807   return "y"
808
809 def printtoomany(name,k,vlist):
810   print "Duplicate in %s: %s:" %(name,k)
811   ct = 0
812   for v in vlist:
813     printtokmention(ct,v)
814     ct = ct + 1
815
816 def checkduplicatetargs(dname,d):
817   targs = d.items()
818   for vi in targs:
819     (k,v) = vi
820     if len(v) > 1:
821       if shouldprintalldups(v) == "y":
822         printtoomany(dname,k,v)
823
824 def checkmissingtarg(name,targ,refs):
825   n = 0
826   rlist = refs.items()
827   for r in rlist:
828     (k,v) = r
829     if targ.has_key(k) == 0:
830        n = n + 1
831        print "Missing target from " + name + ":",k
832   if n == 0: print "No missing targets from " + name +"!"
833
834 def checkmissingref(name,targ,refs):
835   n = 0
836   rlist = targ.items()
837   for r in rlist:
838     (k,v) = r
839     if refs.has_key(k) == 0:
840        n = n + 1
841        print "Unused target from " + name + ":",k
842   if n == 0: print "No unused targets from " + name + "..."
843
844 def print_stats():
845   global dwfnamecommsdict
846   global newcommsdict
847   global targhyperdict
848   global linkhyperdict
849   global indexdict
850   global labeldict
851   global labelrefdict
852   
853   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
854   checkduplicatetargs("commands",newcommsdict)
855   checkduplicatetargs("hypertargets",targhyperdict)
856   checkduplicatetargs("labels",labeldict)
857
858   # Unused targets are nice to know
859   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
860   checkmissingref("labels",labeldict,labelrefdict)
861
862   # Missing targets are a problem to be FIXED
863   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
864   checkmissingtarg("labels",labeldict,labelrefdict)
865
866 # Perhaps these should be controlled by
867 # the command line.
868 debug   = "n"
869
870 def buildfilepaths(files,basetarg):
871   outlist = []
872   prefix = ""
873   for f in files:
874     prefix = ""
875     if len(basetarg) > 0:
876       prefix = basetarg
877     else:
878       prefix = os.path.join("..","latexdoc","")
879     outlist += [prefix + f]
880   return outlist
881 def read_all_args():
882   filelist1 = []
883   filelist2 = []
884   baselist1 = []
885   baselist2 = []
886   basetarg = ""
887   fileio.setkeepordeletecomments("d")
888   if debug == "y":
889     baselist1 = ["testrefer.tex"]
890     baselist2 = ["testrefer.tex"]
891     basetarg = "./"
892   else:
893     baselist1 = ["dwarfnamecmds.tex",
894               "dwarf5.tex",
895               "generaldescription.tex"]
896
897     baselist2 = ["dwarf5.tex",
898               "attributesbytag.tex",
899               "changesummary.tex",
900               "compression.tex",
901               "copyright.tex",
902               "dataobject.tex",
903               "datarepresentation.tex",
904               "debugsectionrelationships.tex",
905               "encodingdecoding.tex",
906               "examples.tex",
907               "foreword.tex",
908               "generaldescription.tex",
909               "gnulicense.tex",
910               "introduction.tex",
911               "otherdebugginginformation.tex",
912               "programscope.tex",
913               "sectionversionnumbers.tex",
914               "splitobjects.tex",
915               "typeentries.tex"]
916   filelist1 = buildfilepaths(baselist1,basetarg)
917   filelist2 = buildfilepaths(baselist2,basetarg)
918
919   if (len(filelist1) < 1) or (len(filelist2) < 1):
920     print >> sys.stderr , "No files specified to refer.py, internal error."
921     sys.exit(1)
922   # Pickup all the newcommand instances.
923   dwf = fileio.readFilelist(filelist1)
924   dwf.dwtransformline(transfunc1)
925
926   # Now find all the uses.
927   dwf2 = fileio.readFilelist(filelist2)
928   dwf2.dwtransformline(transfunc2)
929   print_stats()
930
931 if __name__ == '__main__':
932   read_all_args()
933