refer.py: New app, close to working. To identify
[dwarf-doc.git] / dwarf5 / tools / refer.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12
13 import sys
14 import fileio
15
16 # All the newdwfnamecommands
17 global dwfnamecommsdict
18 # All the newcommand instances.
19 global newcommsdict
20
21 # LINK suffix
22 global linksdict  
23
24 # TARG suffix hyperlink of 'chap:#1' Index #2
25 global targsdict
26 # INDX suffix index #2
27 global indxdict
28 # MARK suffix hypertarget chap:#1  and index #2
29 global markdict
30
31 # NAME suffix. No index, just text shows.
32 global namedict
33
34 # This is the 'normal, most common case in running text.
35 # Default implies index #2 and hyperlink of chap:#1
36 global defaultdict 
37
38 # Targets for \hypertarget and livetarg[i]
39 global targhyperdict
40 # Links using hyperlink.
41 global linkhyperdict
42
43 # label targets (labels)
44 global labeldict
45 # \refersec \ref \vref to labels
46 global labelrefdict
47
48 # The index content for named things.
49 global indexdict
50
51
52 newcommsdict = {}
53 dwfnamecommsdict = {}
54
55 # Links meaning \livelink \livetarg \livetargi macros
56 linksdict  = {}
57 targsdict  = {}
58 indxdict  = {}
59 markdict  = {}
60 namedict  = {}
61 defaultdict  = {}
62 targhyperdict= {}
63 linkhyperdict= {}
64 labeldict = {}
65 labelrefdict = {}
66 indexdict = {}
67
68 global linestoignore
69 linestoignore = []
70
71 # lines_to_ignore is a terrible hack.
72 def add_lines_to_ignore(myfile,lowline,highline):
73   global linestoignore
74   linestoignore += [(myfile._name,lowline,highline)]
75 def in_lines_to_ignore(myfile,line):
76   global linestoignore
77   n = myfile._name
78   for x in linestoignore:
79     (n,l,h) = x
80     if  myfile._name != n:
81       continue
82     if line < l:
83       continue
84     if line >h:
85       continue
86     return "y"
87   return "n"
88    
89
90 # a list of words to ignore: silly stuff.
91 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
92 "Standards":0,"Committee":0,"Version":0 }
93
94 class tokmention:
95   def __init__(self):
96     self._token = '' 
97     self._file = ""
98     self._line = 0
99   def __init__(self,tok,filename,line):
100     self._token = tok
101     self._file = filename
102     self._line = line
103
104 def ischar(tok,c):
105    if tok._class != "ind":
106       return "n"
107    if len(tok._tex) != 1:
108        return "n"
109    if tok._tex[0] != c:
110        return "n"
111    return "y"
112
113 def dwspace(tok):
114   if ischar(tok," ") == "y":
115     return "y"
116   if ischar(tok,"\t") == "y":
117     return "y"
118   return "n"
119   
120   
121 def isbrace(tok,brace):
122   if tok._class != "ind":
123      return "n"
124   if len(tok._tex) != 1:
125      return "n"
126   if brace == tok._tex[0]:
127      return "y"
128   return "n"
129
130 def toknamestring(t):
131   """ Turn a token into its string as a string """
132   return ''.join(t._tex)
133
134
135 def pickup(linetoks,tnumin,pattern,myfile,linenum):
136   """ The token pattern characters are
137   i meaning identifier
138   e meaning identifier, but ifnext token is }
139      we construct an empty identifier for it.
140   [space] meaning whitespace
141   { meaning left brace
142   } meaning right brace
143   * meaning any token except } and end-line
144   
145   Precondition:  linetoks[tnumin] is identifier (meaning a command)
146   Returns: a token list, one per non-space in the pattern.
147      For the *, the token is itself a list of whatever it contains.
148   """
149   outtoks = []
150   numabsorbed = 1
151   inlen = len(linetoks) 
152   curnum = tnumin
153   curtok = linetoks[curnum]
154   patterncharnum = -1
155   for c in pattern:
156     patterncharnum = patterncharnum + 1
157     if curnum >= inlen:
158       print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
159       return outtoks,numabsorbed
160     curtok = linetoks[curnum]
161     if c == " ":
162       while dwspace(curtok) == "y":
163         curnum = curnum + 1
164         if curnum >= inlen:
165           print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
166           return outtoks,numabsorbed
167         numabsorbed = numabsorbed + 1
168         curtok = linetoks[curnum]
169       continue
170     elif c == "i":
171       if curtok._class != "id":
172         print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
173         return outtoks,numabsorbed
174       numabsorbed = numabsorbed + 1
175       outtoks += [curtok]
176       curnum = curnum + 1
177       continue
178     elif c == "e":
179       if curtok._class != "id":
180         if isbrace(curtok,"}") == "y":
181           tk = fileio.dwtoken()
182           tk.insertid("")
183           outtoks += [tk]
184           # Do not update location.
185           continue
186         else:
187           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
188           return outtoks,numabsorbed
189       else: 
190         numabsorbed = numabsorbed + 1
191         outtoks += [curtok]
192         curnum = curnum + 1
193       continue
194     elif c == "{":
195       if isbrace(curtok,"{")  == "y":
196         outtoks += [curtok]
197         curnum = curnum + 1
198         numabsorbed = numabsorbed + 1
199       else:
200         print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
201         return outtoks,numabsorbed
202     elif c == "}":
203       if isbrace(curtok,"}")  == "y":
204         outtoks += [curtok]
205         curnum = curnum + 1
206         numabsorbed = numabsorbed + 1
207       else:
208         print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
209         return outtoks,numabsorbed
210     elif c == "[":
211       if isbrace(curtok,"[")  == "y":
212         outtoks += [curtok]
213         curnum = curnum + 1
214         numabsorbed = numabsorbed + 1
215       else:
216         print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
217         return outtoks,numabsorbed
218     elif c == "]":
219       if isbrace(curtok,"]")  == "y":
220         outtoks += [curtok]
221         curnum = curnum + 1
222         numabsorbed = numabsorbed + 1
223       else:
224         print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
225         return outtoks,numabsorbed
226     elif c == "*":
227       outlist = []
228       curtok = linetoks[curnum]
229       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
230         if dwspace(curtok) == "n":
231            outlist += [curtok]
232         curnum = curnum + 1
233         if curnum >= inlen:
234           outtoks += [outlist]
235           if patterncharnum < (len(pattern) -1): 
236             print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
237           return outtoks,numabsorbed
238         numabsorbed = numabsorbed + 1
239         curtok = linetoks[curnum]
240       # Found a right brace, so done here.
241       outtoks += [outlist]
242     else:
243         print "ERROR pattern had unexpected character ",pattern
244         sys.exit(1)
245   return outtoks,numabsorbed
246
247 def printbadcommand(name,myfile,myline):
248   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
249
250 def applytodict(d,k,v):
251   keystring = k
252   if d.has_key(keystring) == 0:
253      d[keystring] =  [v]
254   else:
255      # This is a duplicate entry.
256      # We will report on it later as appropriate.
257      existing = d.get(keystring)
258      existing += [v]
259      d[keystring] =  existing
260
261 # See how many "{" there are on the line.
262 # return the count.
263 def countbraces(linetoks,tnumin):
264   lasttoknum = len(linetoks) -1
265   lb = 0
266   while tnumin < lasttoknum:
267     x = linetoks[tnumin] 
268     if x._class == "ind":
269        n = toknamestring(x)
270        if n == "{":
271           lb = lb + 1
272     tnumin = tnumin + 1
273   return lb
274      
275
276
277 def processbegin(linetoks,tnumin,myfile,linenum):
278   global targhyperdict
279   global indexdict
280   lbracecount = countbraces(linetoks,tnumin)
281   if lbracecount < 3:
282     return 1
283   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum)
284   if len(ourtoks) < 12:
285     return 1
286   lcom = ourtoks[2]
287   lcomname = toknamestring(lcom)
288   if lcomname != "simplenametable":
289     return inlen
290   targ = ourtoks[11]
291   hypstr = toknamestring(targ)
292   hypmen = tokmention(targ,myfile,linenum)
293   applytodict(labeldict,hypstr,hypmen)
294   return inlen
295
296 # If justlink == "y" this is a hypertarget command literally.
297 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
298   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
299   global targhyperdict
300   global indexdict
301   t = linetoks[tnumin]
302   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum)
303   if len(ourtoks) > 5:
304     t2 = ourtoks[2];
305     index = tokmention(t2,myfile,linenum)
306     name = toknamestring(t2)
307     applytodict(targhyperdict,name,index)
308     if justlink == "n":
309       t2 = ourtoks[5];
310       # Ignore for now.
311       #name = toknamestring(t2)
312       #if len(name) > 0:
313       #  index = tokmention(t2,myfile,linenum)
314       #  applytodict(indexdict,name,index)
315   else:
316     tn = toknamestring(linetoks[tnumin])
317     printbadcommand(tn,myfile,linenum)
318   return inlen
319 def livetargiprocess(linetoks,tnumin,myfile,linenum):
320   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
321   global targhyperdict
322   global indexdict
323   t = linetoks[tnumin]
324   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { e } { * }",myfile,linenum)
325   if len(ourtoks) > 5:
326     t2 = ourtoks[2];
327     index = tokmention(t2,myfile,linenum)
328     name = toknamestring(t2)
329     applytodict(targhyperdict,name,index)
330
331     t2 = ourtoks[5];
332     name = toknamestring(t2)
333     if len(name) > 0:
334       index = tokmention(t2,myfile,linenum)
335       applytodict(indexdict,name,index)
336   else:
337     tn = toknamestring(linetoks[tnumin])
338     printbadcommand(tn,myfile,linenum)
339   return inlen
340 # if justlink == "y" it is a plain hyperlinkcommand
341 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
342   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
343   global linkhyperdict
344   global indexdict
345   t = linetoks[tnumin]
346   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum)
347   if len(ourtoks) > 5:
348     t2 = ourtoks[2];
349     index = tokmention(t2,myfile,linenum)
350     name = toknamestring(t2)
351     applytodict(linkhyperdict,name,index)
352
353     # can be multiword. For now do not bother with every index.
354     #t2 = ourtoks[5];
355     #index = tokmention(t2,myfile,linenum)
356     #name = toknamestring(t2)
357     #applytodict(indexdict,name,index)
358   else:
359     tn = toknamestring(linetoks[tnumin])
360     printbadcommand(tn,myfile,linenum)
361   return inlen
362 def labelprocess(linetoks,tnumin,myfile,linenum):
363   """ \label{alabel} """
364   global labeldict
365   t = linetoks[tnumin]
366   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
367   if len(ourtoks) > 2:
368     t2 = ourtoks[2];
369     index = tokmention(t2,myfile,linenum)
370     name = toknamestring(t2)
371     applytodict(labeldict,name,index)
372   else:
373     tn = toknamestring(linetoks[tnumin])
374     printbadcommand(tn,myfile,linenum)
375
376   return inlen
377 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
378   """ \addtoindex{strings} """
379   global indexdict
380   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum)
381   if len(ourtoks) > 2:
382     # The * means a list of tokens.
383     fake = ""
384     #t2 = ourtoks[2];
385     #index = tokmention(t2,myfile,linenum)
386     #name = toknamestring(t2)
387     #applytodict(indexdict,name,index)
388   else:
389     tn = toknamestring(linetoks[tnumin])
390     printbadcommand(tn,myfile,linenum)
391   return inlen
392 def hyperlinkname(name,tnumin,myfile,linenum):
393   global linkhyperdict
394   tkmod = fileio.dwtoken()
395   tkmod.insertid(name)
396   tm = tokmention(tkmod,myfile,linenum)
397   applytodict(linkhyperdict,name,tm)
398   return 1
399 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
400   """ \hyperlink{entryname} """
401   global linkhyperdict
402   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
403   if len(ourtoks) > 2:
404     t2 = ourtoks[2];
405     index = tokmention(t2,myfile,linenum)
406     name = toknamestring(t2)
407     applytodict(linkhyperdict,name,index)
408   else:
409     tn = toknamestring(linetoks[tnumin])
410     printbadcommand(tn,myfile,linenum)
411   return inlen
412
413 def indexprocess(linetoks,tnumin,myfile,linenum):
414   """ \index{indexentryname} """
415   global indexdict
416   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum)
417   if len(ourtoks) > 2:
418     fake = ""
419     # For now not bothering with index strings
420     #t2 = ourtoks[2];
421     #index = tokmention(t2,myfile,linenum)
422     #name = toknamestring(t2)
423     #applytodict(indexdict,name,index)
424   else:
425     tn = toknamestring(linetoks[tnumin])
426     printbadcommand(tn,myfile,linenum)
427   return inlen
428 def refersecprocess(linetoks,tnumin,myfile,linenum):
429   """ \refersec{label} """
430   global labelrefdict
431   t = linetoks[tnumin]
432   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
433   if len(ourtoks) > 2:
434     t2 = ourtoks[2];
435     index = tokmention(t2,myfile,linenum)
436     name = toknamestring(t2)
437     applytodict(labelrefdict,name,labelrefdict)
438   else:
439     tn = toknamestring(linetoks[tnumin])
440     printbadcommand(tn,myfile,linenum)
441   return inlen
442
443 def firstnonblank(linetoks):
444   tnum = 0
445   lasttoknum = len(linetoks)
446   while tnum < lasttoknum:
447     x = linetoks[tnum]
448     if x._class != "ind":
449       return tnum
450     if toknamestring(x) == " ":
451       tnum = tnum + 1
452       continue
453     elif toknamestring(x) == "\t":
454       tnum = tnum + 1
455       continue
456     return tnum 
457   return tnum
458 # Deals solely with finding new commands.
459 # This done as a first pass so we can recognize when tokens are
460 # really commands, something transfunc2, the second pass, 
461 # wants to know. 
462 def transfunc1(linetoks,myfile,linenum):
463   global dwfnamecommsdict
464   global newcommsdict
465
466   if len(linetoks) < 1:
467     return linetoks
468   tnum = firstnonblank(linetoks)
469   if tnum >= len(linetoks):
470     return linetoks
471   initialtok = linetoks[tnum]
472   itokstring=toknamestring(initialtok)
473   if itokstring == "\\expandafter\\def\\csname":
474     return linetoks
475   if in_lines_to_ignore(myfile,linenum) == "y":
476     return linetoks
477   if itokstring == "\\newcommand":
478     t1 = linetoks[tnum+1]
479     if not isbrace(t1,'{'):
480       print "Improper character in newcommand", myfile,linenum
481       sys.exit(1)
482     t2 = linetoks[tnum+2]
483     if toknamestring(t2) == "\\simplenametablerule":
484        add_lines_to_ignore(myfile,linenum,linenum+18)
485     if toknamestring(t2) != "\\newdwfnamecommands":
486        tm = tokmention(t2,myfile,linenum)
487        applytodict(newcommsdict,toknamestring(t2),tm)
488     #Be silent on newdwfnamecommands, it is normal.
489     #else:
490     #   print "newcommand on newdwfnamecommands ignored intentionally."
491     return linetoks
492   elif itokstring == "\\newdwfnamecommands":
493     t1 = linetoks[tnum+1]
494     if not isbrace(t1,'{'):
495        print "Improper character in newdwfnamecommands", myfile._name,linenum
496        sys.exit(1)
497     # The token name string will be DWsomething and we want
498     # The token to appear as \DWsomething as that is how references
499     # The usages determine what secondary actions are applied.
500     # are coded.
501     t2 = linetoks[tnum+2]
502     tkmod = fileio.dwtoken()
503     tkmod.insertid("\\" + toknamestring(t2))
504     tm = tokmention(tkmod,myfile,linenum)
505     applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
506     return linetoks
507   return linetoks
508
509
510 def delsuffix(n,suf):
511   slen = len(suf)
512   nlen = len(n)
513   lastcharnum = nlen - slen
514   outstring = n[0:lastcharnum]
515   return outstring
516 def deloptionalprefix(n,pref):
517   if not n.startswith(pref):
518     return n
519   plen = len(pref)
520   nlen = len(n)
521   outstring = n[plen:nlen]
522   return outstring
523
524 def printodderr(rawname,comname,myfile,linenum):
525   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
526
527 def rawnameiscommand(t,suff):
528    if not t.startswith("\\DW"):
529      return ""
530    if not t.endswith(suff):
531      return ""
532    commandname = delsuffix(t,suff)
533    return commandname
534
535 # Delete any leading backslash.
536 # Prefix the result with chap:
537 def makelinkname(t):
538    s = deloptionalprefix(t,"\\");
539    s2 = "chap:" + s
540    return s2;
541    
542
543 # Assumes all new commands known already.
544 # This deals with targets and links (various flavors).
545 def transfunc2(linetoks,myfile,linenum):
546   global newcommsdict
547   global dwfnamecommsdict
548   global newcommsdict
549
550   # Link naming target
551   global linksdict
552   # TARG suffix
553   global targsdict
554   # INDX suffix
555   global indxdict
556   # MARK suffix
557   global markdict
558   # NAME suffix
559   global namedict
560   # This is the 'normal, most common case in running text.
561   global defaultdict
562
563   if len(linetoks) < 1:
564     return linetoks
565   if in_lines_to_ignore(myfile,linenum) == "y":
566     return linetoks
567   initialtok = linetoks[0]
568   itokstring=toknamestring(initialtok)
569   # Skip all the newcommand stuff.
570   if itokstring == "\\newcommand":
571     return linetoks
572   elif itokstring == '\\newdwfnamecommands':
573     return linetoks
574
575   # Now deal with a regular line.
576
577   tnumin = 0
578   changes = 0
579   lasttoknum = len(linetoks) -1
580   for x in linetoks:
581     if int(tnumin) > int(lasttoknum):
582       break
583     t = linetoks[tnumin]
584     if t._class != "id":
585       tnumin = tnumin + 1
586       continue
587     rawname = toknamestring(t)
588     commandname=""
589     #rawnameiscommand(rawname,"",basecommand)
590     if rawname == "\\expandafter\\def\\csname":
591       return linetoks
592     if rawname == "\\begin":
593       tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
594       tnumin = tnumin + tnumcount
595       continue
596     if dwfnamecommsdict.has_key(rawname):
597       # We know this one. 
598       # It is a default case name reference.
599       # index the DWname
600       # Link is to chap:DWname
601       tm = tokmention(t,myfile,linenum)
602       applytodict(defaultdict,rawname,tm)
603       linkname = makelinkname(rawname)
604       indxname = deloptionalprefix(commandname,"\\")
605       applytodict(linkhyperdict,linkname,tm);
606       applytodict(indexdict,indxname,tm);
607       tnumin = tnumin + 1
608       continue
609     if  newcommsdict.has_key(rawname):
610       # We know this one. We have to see what it is
611       # To decide what to do.
612       # some DWOPbreg*  DWOPreg*   and MDfive are special.
613       # A variety of other such defined commands are irrelevant to us here.
614
615       tnumcount = 1
616       if rawname == "\\livetarg":
617         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
618       elif rawname == "\\livetargi":
619         tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
620       elif rawname == "\\livelink":
621         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
622       elif rawname == "\\livelinki":
623         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
624       #elif rawname == "\\label":
625       #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
626       elif rawname == "\\refersec":
627         # does \ref
628         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
629       elif rawname == "\\referfol":
630         # does \vref from varioref package
631         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
632       elif rawname == "\\index":
633         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
634       elif rawname == "\\addtoindex":
635         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
636       elif rawname == "\\addtoindexx":
637         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
638       elif rawname == "\\addttindex":
639         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
640       elif rawname == "\\addttindexx":
641         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
642       elif rawname == "\\DWOPbregtwo":
643         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
644       elif rawname == "\\DWOPbregthree":
645         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
646       elif rawname == "\\DWOPbregfour":
647         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
648       elif rawname == "\\DWOPbregfive":
649         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
650       elif rawname == "\\DWOPbregeleven":
651         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
652       elif rawname == "\\MDfive":
653         tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
654       else:
655         fake = ""
656         # If missing anything important, perhaps turn ths on.
657         #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
658       tnumin = tnumin + tnumcount
659       continue
660     # Suffixes are LINK TARG INDX MARK NAME
661     commandname =rawnameiscommand(rawname,"LINK")
662     if len(commandname) > 0:
663       # We do not expect to see this in text, LINK is really how
664       # the defaultdict entries are created so should
665       # be hidden from us here.
666       # index the DWname
667       # Link is to chap:DWname
668       if dwfnamecommsdict.has_key(commandname):
669         tm = tokmention(t,myfile,linenum)
670         linkname = makelinkname(commandname)
671         indxname = deloptionalprefix(commandname,"\\")
672         applytodict(linkhyperdict,linkname,tm)
673         applytodict(indexdict,indxname,tm);
674       else:
675         printodderr(rawname,commandname,myfile,linenum)
676       tnumin = tnumin + 1
677       continue
678     commandname =rawnameiscommand(rawname,"TARG")
679     if len(commandname) > 0:
680       # index DWname
681       # Set chap:DWname as having target defined
682       if dwfnamecommsdict.has_key(commandname):
683         tm = tokmention(t,myfile,linenum)
684         targname = makelinkname(commandname)
685         indxname = deloptionalprefix(commandname,"\\")
686         applytodict(targsdict,targname,tm)
687         applytodict(indexdict,indxname,tm);
688       else:
689         printodderr(rawname,commandname,myfile,linenum)
690       tnumin = tnumin + 1
691       continue
692     commandname =rawnameiscommand(rawname,"INDX")
693     if len(commandname) > 0:
694       # Index DWname
695       if dwfnamecommsdict.has_key(commandname):
696         tm = tokmention(t,myfile,linenum)
697         indexname = deloptionalprefix(commandname,"\\")
698         applytodict(indxdict,indexname,tm)
699       else:
700         printodderr(rawname,commandname,myfile,linenum)
701       tnumin = tnumin + 1
702       continue
703     commandname =rawnameiscommand(rawname,"MARK")
704     if len(commandname) > 0:
705       # set chap:DWname as target defined
706       # index DWname
707       if dwfnamecommsdict.has_key(commandname):
708         tm = tokmention(t,myfile,linenum)
709         applytodict(markdict,commandname,tm)
710         indexname = deloptionalprefix(commandname,"\\")
711         applytodict(indxdict,indexname,tm)
712         targname = makelinkname(commandname)
713         applytodict(markdict,targname,tm)
714       else:
715         printodderr(rawname,commandname,myfile,linenum)
716       tnumin = tnumin + 1
717       continue
718     commandname =rawnameiscommand(rawname,"NAME")
719     if len(commandname) > 0:
720       # No actions with NAME (but put in namedict anyway).
721       if dwfnamecommsdict.has_key(commandname):
722         tm = tokmention(t,myfile,linenum)
723         applytodict(namedict,commandname,tm)
724       else:
725         printodderr(rawname,commandname,myfile,linenum)
726       tnumin = tnumin + 1
727       continue
728     if rawname == "\\label":
729       # This is a builtin, not our newcommand.
730       tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
731       tnumin = tnumin + tnumcount
732       continue
733     if rawname == "\\hypertarget":
734       # This is a builtin, not our newcommand.
735       tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
736       tnumin = tnumin + tnumcount
737       continue
738     if rawname == "\\hyperlink":
739       # This is a builtin, not our newcommand.
740       tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
741       tnumin = tnumin + tnumcount
742       continue
743     # Some random data or text here.
744
745     tnumin = tnumin + 1
746     # We don't know what this is. Probably ok?
747     # End of for loop.
748   return linetoks
749
750 def sort_tokmlist(mylist):
751   aux = [ (''.join(x._token._tex),x) for x in mylist ]
752   aux.sort()
753   return[ (x[1]) for x in aux]
754
755 def printtokmention(ct,v):
756   n = v._token
757   name =  toknamestring(n)
758   f = v._file
759   l = v._line
760   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
761
762 def printtoomany(name,k,vlist):
763   print "Duplicate in %s: %s:" %(name,k)
764   ct = 0
765   for v in vlist:
766       printtokmention(ct,v)
767       ct = ct + 1
768
769 def checkduplicatetargs(dname,d):
770   targs = d.items()
771   for vi in targs:
772     (k,v) = vi
773     if len(v) > 1:
774       printtoomany(dname,k,v)
775
776 def checkmissingtarg(name,targ,refs):
777   rlist = refs.items()
778   for r in rlist:
779     (k,v) = r
780     if targ.has_key(k) == 0:
781        print "target missing from",name,":",k
782
783 def checkmissingref(name,targ,refs):
784   rlist = targ.items()
785   for r in rlist:
786     (k,v) = r
787     if refs.has_key(k) == 0:
788        print "Unused target from",name,":",k
789   
790 def print_stats():
791   # All the newdwfnamecommands
792   global dwfnamecommsdict
793   # All the newcommand instances.
794   global newcommsdict
795   
796   # LINK suffix
797   global linksdict
798   
799   # TARG suffix hyperlink of 'chap:#1' Index #2
800   global targsdict
801   # INDX suffix index #2
802   global indxdict
803   # MARK suffix hypertarget chap:#1  and index #2
804   global markdict
805   # NAME suffix. No index, just text shows.
806   global namedict
807   # This is the 'normal, most common case in running text.
808   # Default implies index #2 and hyperlink of chap:#1
809   global defaultdict
810   
811   # Targets for \hypertarget and livetarg[i]
812   global targhyperdict
813   # Links using hyperlink.
814   global linkhyperdict
815   # label targets (labels)
816   global labeldict
817   # \refersec \ref \vref to labels
818   global labelrefdict
819   # The index content for named things.
820   global indexdict
821   
822   checkduplicatetargs("targets",targsdict)
823   checkduplicatetargs("hypertargets",targhyperdict)
824   checkduplicatetargs("MARKs",markdict)
825   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
826   checkduplicatetargs("commands",newcommsdict)
827   checkduplicatetargs("labels",labeldict)
828
829   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
830   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
831
832   checkmissingtarg("labels",labeldict,labelrefdict)
833   checkmissingref("labels",labeldict,labelrefdict)
834
835   checkmissingref("names",dwfnamecommsdict,namedict)
836
837 debug = "n"
838  
839 def read_all_args():
840   filelist1 = []
841   filelist2 = []
842   fileio.setkeepordeletecomments("d")
843   filelist1 = {"./testrefer.tex"}
844   filelist2 = {"./testrefer.tex"}
845   if debug == "n":
846     filelist1 = {"../latexdoc/dwarfnamecmds.tex",
847               "../latexdoc/dwarf5.tex",
848               "../latexdoc/generaldescription.tex"}
849
850     filelist2 = {"../latexdoc/dwarf5.tex",
851               "../latexdoc/attributesbytag.tex",
852               "../latexdoc/changesummary.tex",
853               "../latexdoc/compression.tex",
854               "../latexdoc/copyright.tex",
855               "../latexdoc/dataobject.tex",
856               "../latexdoc/datarepresentation.tex",
857               "../latexdoc/debugsectionrelationships.tex",
858               "../latexdoc/encodingdecoding.tex",
859               "../latexdoc/examples.tex",
860               "../latexdoc/foreword.tex",
861               "../latexdoc/generaldescription.tex",
862               "../latexdoc/gnulicense.tex",
863               "../latexdoc/introduction.tex",
864               "../latexdoc/otherdebugginginformation.tex",
865               "../latexdoc/programscope.tex",
866               "../latexdoc/sectionversionnumbers.tex",
867               "../latexdoc/splitobjects.tex",
868               "../latexdoc/typeentries.tex"}
869
870   if (len(filelist1) < 1) or (len(filelist2) < 1):
871     print >> sys.stderr , "No files specified to refer.py, internal error."
872     sys.exit(1)
873   # Pickup all the newcommand instances.
874   dwf = fileio.readFilelist(filelist1)
875   dwf.dwtransformline(transfunc1)
876
877   # Now find all the uses.
878   dwf2 = fileio.readFilelist(filelist2)
879   dwf2.dwtransformline(transfunc2)
880   print_stats()
881
882 if __name__ == '__main__':
883   read_all_args()
884