a954d45efe041c9f6b4505fc60e99d2fa477d5fc
[dwarf-doc.git] / dwarf5 / tools / refer.py
1 # Copyright 2014 DWARF Debugging Information Format Committee
2 #
3 # Looks at dwarfnamecmds.tex(+) to find all the commands
4 # And sees what is actually used and whether references
5 # have definitions.
6 # The initial implemenation just *assumes* it is run from the tools
7 # directory and the file names are built in to the source here.
8 #
9 # Run as (for example)
10 #   python refer.py 
11
12 # This is the simplest 'parse' of the .tex that we can manage
13 # while still finding what we want to find.
14 # One would hardly call it a parser, really.
15
16 # There are essentially three namespaces at present in the document.
17 # The hyperlink/hypertarget namespace.
18 # The label  vref ref namespace
19 #   which also involves  our refersec referfol.
20 # The index namespace, which we are not presently filling in very much. 
21
22 import sys
23 import os
24 import os.path
25 import fileio
26
27 # These two hold the commands we care about so we can
28 # bypass most lines easily in phase two.
29 # All the newdwfnamecommands
30 global dwfnamecommsdict
31 # All the newcommand instances.
32 global newcommsdict
33
34 # NAME suffix. No index, just text shows.
35 # Not very useful so far
36 global namedict
37
38
39 # Targets for \hypertarget and \hyperlink
40 #targhyperdict is all the \hyperarget instances.
41 #linkhyperdict is all the \hyperlink instances.
42 global targhyperdict
43 global linkhyperdict
44
45 # label targets (labels)
46 global labeldict
47 # \refersec \ref \vref to labels
48 global labelrefdict
49
50 # The index content for named things.
51 # The strings here are the links, the targets are
52 # built by the latex index software.
53 # So this dictionary is not really needed here (yet)
54 # and is not yet fully built up.
55 global indexdict
56
57
58 newcommsdict = {}
59 dwfnamecommsdict = {}
60
61 # Links meaning \livelink \livetarg \livetargi macros
62 #linksdict  = {}
63 #targsdict  = {}
64 namedict  = {}
65 targhyperdict= {}
66 linkhyperdict= {}
67 labeldict = {}
68 labelrefdict = {}
69 indexdict = {}
70
71 global linestoignore
72 linestoignore = []
73
74 # lines_to_ignore is a terrible hack.
75 def add_lines_to_ignore(myfile,lowline,highline):
76   global linestoignore
77   linestoignore += [(myfile._name,lowline,highline)]
78 def in_lines_to_ignore(myfile,line):
79   global linestoignore
80   n = myfile._name
81   for x in linestoignore:
82     (n,l,h) = x
83     if  myfile._name != n:
84       continue
85     if line < l:
86       continue
87     if line >h:
88       continue
89     return "y"
90   return "n"
91    
92
93 # a list of words to ignore: silly stuff.
94 ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
95 "Standards":0,"Committee":0,"Version":0 }
96
97 class tokmention:
98   def __init__(self):
99     self._token = '' 
100     self._file = ""
101     self._line = 0
102   def __init__(self,tok,filename,line):
103     self._token = tok
104     self._file = filename
105     self._line = line
106
107 def ischar(tok,c):
108    if tok._class != "ind":
109       return "n"
110    if len(tok._tex) != 1:
111        return "n"
112    if tok._tex[0] != c:
113        return "n"
114    return "y"
115
116 def dwspace(tok):
117   if ischar(tok," ") == "y":
118     return "y"
119   if ischar(tok,"\t") == "y":
120     return "y"
121   return "n"
122   
123   
124 def isbrace(tok,brace):
125   if tok._class != "ind":
126      return "n"
127   if len(tok._tex) != 1:
128      return "n"
129   if brace == tok._tex[0]:
130      return "y"
131   return "n"
132
133 def toknamestring(t):
134   """ Turn a token into its string as a string """
135   return ''.join(t._tex)
136
137
138 def pickup(linetoks,tnumin,pattern,myfile,linenum,suppresserr):
139   """ The token pattern characters are
140   i meaning identifier
141   e meaning identifier, but ifnext token is }
142      we construct an empty identifier for it.
143   [space] meaning whitespace
144   { meaning left brace
145   } meaning right brace
146   * meaning any token except } and end-line
147   
148   Precondition:  linetoks[tnumin] is identifier (meaning a command)
149   Returns: a token list, one per non-space in the pattern.
150      For the *, the token is itself a list of whatever it contains.
151   """
152   outtoks = []
153   numabsorbed = 1
154   inlen = len(linetoks) 
155   curnum = tnumin
156   curtok = linetoks[curnum]
157   patterncharnum = -1
158   for c in pattern:
159     patterncharnum = patterncharnum + 1
160     if curnum >= inlen:
161       if suppresserr == "n":
162         print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
163       return outtoks,numabsorbed
164     curtok = linetoks[curnum]
165     if c == " ":
166       while dwspace(curtok) == "y":
167         curnum = curnum + 1
168         if curnum >= inlen:
169           if suppresserr == "n":
170             print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
171           return outtoks,numabsorbed
172         numabsorbed = numabsorbed + 1
173         curtok = linetoks[curnum]
174       continue
175     elif c == "i":
176       if curtok._class != "id":
177         if suppresserr == "n":
178           print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
179         return outtoks,numabsorbed
180       numabsorbed = numabsorbed + 1
181       outtoks += [curtok]
182       curnum = curnum + 1
183       continue
184     elif c == "e":
185       if curtok._class != "id":
186         if isbrace(curtok,"}") == "y":
187           tk = fileio.dwtoken()
188           tk.insertid("")
189           outtoks += [tk]
190           # Do not update location.
191           continue
192         else:
193           if suppresserr == "n":
194             print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
195           return outtoks,numabsorbed
196       else: 
197         numabsorbed = numabsorbed + 1
198         outtoks += [curtok]
199         curnum = curnum + 1
200       continue
201     elif c == "{":
202       if isbrace(curtok,"{")  == "y":
203         outtoks += [curtok]
204         curnum = curnum + 1
205         numabsorbed = numabsorbed + 1
206       else:
207         if suppresserr == "n":
208           print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
209         return outtoks,numabsorbed
210     elif c == "}":
211       if isbrace(curtok,"}")  == "y":
212         outtoks += [curtok]
213         curnum = curnum + 1
214         numabsorbed = numabsorbed + 1
215       else:
216         if suppresserr == "n":
217           print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
218         return outtoks,numabsorbed
219     elif c == "[":
220       if isbrace(curtok,"[")  == "y":
221         outtoks += [curtok]
222         curnum = curnum + 1
223         numabsorbed = numabsorbed + 1
224       else:
225         if suppresserr == "n":
226           print "ERROR line  expected [  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
227         return outtoks,numabsorbed
228     elif c == "]":
229       if isbrace(curtok,"]")  == "y":
230         outtoks += [curtok]
231         curnum = curnum + 1
232         numabsorbed = numabsorbed + 1
233       else:
234         if suppresserr == "n":
235           print "ERROR line  expected ]  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
236         return outtoks,numabsorbed
237     elif c == "*":
238       outlist = []
239       curtok = linetoks[curnum]
240       while isbrace(curtok,"}") == "n" and isbrace(curtok,"]") == "n":
241         if dwspace(curtok) == "n":
242            outlist += [curtok]
243         curnum = curnum + 1
244         if curnum >= inlen:
245           outtoks += [outlist]
246           if patterncharnum < (len(pattern) -1): 
247             if suppresserr == "n":
248               print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
249           return outtoks,numabsorbed
250         numabsorbed = numabsorbed + 1
251         curtok = linetoks[curnum]
252       # Found a right brace, so done here.
253       outtoks += [outlist]
254     else:
255         if suppresserr == "n":
256           print "ERROR pattern had unexpected character ",pattern
257   return outtoks,numabsorbed
258
259 def printbadcommand(name,myfile,myline):
260   print "Error: command %s missing operand file %s line %d" %(name,myfile._name,myline)
261
262 def applytodict(d,k,v):
263   keystring = k
264   if d.has_key(keystring) == 0:
265      d[keystring] =  [v]
266   else:
267      # This is a duplicate entry.
268      # We will report on it later as appropriate.
269      existing = d.get(keystring)
270      existing += [v]
271      d[keystring] =  existing
272
273 # See how many "{" there are on the line.
274 # return the count.
275 def countbraces(linetoks,tnumin):
276   lasttoknum = len(linetoks) -1
277   lb = 0
278   while tnumin < lasttoknum:
279     x = linetoks[tnumin] 
280     if x._class == "ind":
281        n = toknamestring(x)
282        if n == "{":
283           lb = lb + 1
284     tnumin = tnumin + 1
285   return lb
286      
287
288
289 # Here we try two different parses, the [] is optional
290 # with simplenametable.
291 def processbegin(linetoks,tnumin,myfile,linenum):
292   global targhyperdict
293   global indexdict
294   lbracecount = countbraces(linetoks,tnumin)
295   if lbracecount < 3:
296     return 1
297   ourtoks,inlen = pickup(linetoks,tnumin," i { i } [ * ] { * } { i }",myfile,linenum,"y")
298   if len(ourtoks) == 13:
299     lcom = ourtoks[2]
300     lcomname = toknamestring(lcom)
301     if lcomname != "simplenametable":
302       return inlen
303     targ = ourtoks[11]
304     hypstr = toknamestring(targ)
305     hypmen = tokmention(targ,myfile,linenum)
306     applytodict(labeldict,hypstr,hypmen)
307     return inlen
308   ourtoks,inlen = pickup(linetoks,tnumin," i { i } { * } { i }",myfile,linenum,"y")
309   if len(ourtoks) < 10:
310       return inlen
311   lcom = ourtoks[2]
312   lcomname = toknamestring(lcom)
313   if lcomname != "simplenametable":
314       return inlen
315   targ = ourtoks[8]
316   hypstr = toknamestring(targ)
317   hypmen = tokmention(targ,myfile,linenum)
318   applytodict(labeldict,hypstr,hypmen)
319   return inlen
320
321 # If justlink == "y" this is a hypertarget command literally.
322 def livetargprocess(linetoks,tnumin,myfile,linenum,justlink):
323   """ \livetarg{chap:DWTAGtemplatevalueparameter}{DWTAGtemplatevalueparameter} """
324   global targhyperdict
325   global indexdict
326   t = linetoks[tnumin]
327   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
328   if len(ourtoks) > 5:
329     t2 = ourtoks[2];
330     index = tokmention(t2,myfile,linenum)
331     name = toknamestring(t2)
332     applytodict(targhyperdict,name,index)
333     if justlink == "n":
334       t2 = ourtoks[5];
335       # Ignore for now.
336       #name = toknamestring(t2)
337       #if len(name) > 0:
338       #  index = tokmention(t2,myfile,linenum)
339       #  applytodict(indexdict,name,index)
340   else:
341     tn = toknamestring(linetoks[tnumin])
342     printbadcommand(tn,myfile,linenum)
343   return inlen
344 def livetargiprocess(linetoks,tnumin,myfile,linenum):
345   """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
346   global targhyperdict
347   global indexdict
348   t = linetoks[tnumin]
349   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { e } { * }",myfile,linenum,"n")
350   if len(ourtoks) > 5:
351     t2 = ourtoks[2];
352     index = tokmention(t2,myfile,linenum)
353     name = toknamestring(t2)
354     applytodict(targhyperdict,name,index)
355
356     t2 = ourtoks[5];
357     name = toknamestring(t2)
358     if len(name) > 0:
359       index = tokmention(t2,myfile,linenum)
360       applytodict(indexdict,name,index)
361   else:
362     tn = toknamestring(linetoks[tnumin])
363     printbadcommand(tn,myfile,linenum)
364   return inlen
365 # if justlink == "y" it is a plain hyperlinkcommand
366 def livelinkprocess(linetoks,tnumin,myfile,linenum,justlink):
367   """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
368   global linkhyperdict
369   global indexdict
370   t = linetoks[tnumin]
371   ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { * }",myfile,linenum,"n")
372   if len(ourtoks) > 5:
373     t2 = ourtoks[2];
374     index = tokmention(t2,myfile,linenum)
375     name = toknamestring(t2)
376     applytodict(linkhyperdict,name,index)
377
378     # can be multiword. For now do not bother with every index.
379     #t2 = ourtoks[5];
380     #index = tokmention(t2,myfile,linenum)
381     #name = toknamestring(t2)
382     #applytodict(indexdict,name,index)
383   else:
384     tn = toknamestring(linetoks[tnumin])
385     printbadcommand(tn,myfile,linenum)
386   return inlen
387 def labelprocess(linetoks,tnumin,myfile,linenum):
388   """ \label{alabel} """
389   global labeldict
390   t = linetoks[tnumin]
391   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
392   if len(ourtoks) > 2:
393     t2 = ourtoks[2];
394     index = tokmention(t2,myfile,linenum)
395     name = toknamestring(t2)
396     applytodict(labeldict,name,index)
397   else:
398     tn = toknamestring(linetoks[tnumin])
399     printbadcommand(tn,myfile,linenum)
400
401   return inlen
402 def addtoindexprocess(linetoks,tnumin,myfile,linenum):
403   """ \addtoindex{strings} """
404   global indexdict
405   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
406   if len(ourtoks) > 2:
407     # The * means a list of tokens.
408     fake = ""
409     #t2 = ourtoks[2];
410     #index = tokmention(t2,myfile,linenum)
411     #name = toknamestring(t2)
412     #applytodict(indexdict,name,index)
413   else:
414     tn = toknamestring(linetoks[tnumin])
415     printbadcommand(tn,myfile,linenum)
416   return inlen
417 def hyperlinkname(name,tnumin,myfile,linenum):
418   global linkhyperdict
419   tkmod = fileio.dwtoken()
420   tkmod.insertid(name)
421   tm = tokmention(tkmod,myfile,linenum)
422   applytodict(linkhyperdict,name,tm)
423   return 1
424 def hyperlinkprocess(linetoks,tnumin,myfile,linenum):
425   """ \hyperlink{entryname} """
426   global linkhyperdict
427   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
428   if len(ourtoks) > 2:
429     t2 = ourtoks[2];
430     index = tokmention(t2,myfile,linenum)
431     name = toknamestring(t2)
432     applytodict(linkhyperdict,name,index)
433   else:
434     tn = toknamestring(linetoks[tnumin])
435     printbadcommand(tn,myfile,linenum)
436   return inlen
437
438 def indexprocess(linetoks,tnumin,myfile,linenum):
439   """ \index{indexentryname} """
440   global indexdict
441   ourtoks,inlen = pickup(linetoks,tnumin,"i { * }",myfile,linenum,"n")
442   if len(ourtoks) > 2:
443     fake = ""
444     # For now not bothering with index strings
445     #t2 = ourtoks[2];
446     #index = tokmention(t2,myfile,linenum)
447     #name = toknamestring(t2)
448     #applytodict(indexdict,name,index)
449   else:
450     tn = toknamestring(linetoks[tnumin])
451     printbadcommand(tn,myfile,linenum)
452   return inlen
453 def refersecprocess(linetoks,tnumin,myfile,linenum):
454   """ \refersec{label} """
455   global labelrefdict
456   t = linetoks[tnumin]
457   ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum,"n")
458   if len(ourtoks) > 2:
459     t2 = ourtoks[2];
460     index = tokmention(t2,myfile,linenum)
461     name = toknamestring(t2)
462     applytodict(labelrefdict,name,labelrefdict)
463   else:
464     tn = toknamestring(linetoks[tnumin])
465     printbadcommand(tn,myfile,linenum)
466   return inlen
467
468 def firstnonblank(linetoks):
469   tnum = 0
470   lasttoknum = len(linetoks)
471   while tnum < lasttoknum:
472     x = linetoks[tnum]
473     if x._class != "ind":
474       return tnum
475     if toknamestring(x) == " ":
476       tnum = tnum + 1
477       continue
478     elif toknamestring(x) == "\t":
479       tnum = tnum + 1
480       continue
481     return tnum 
482   return tnum
483 # Deals solely with finding new commands.
484 # This done as a first pass so we can recognize when tokens are
485 # really commands, something transfunc2, the second pass, 
486 # wants to know. 
487 def transfunc1(linetoks,myfile,linenum):
488   global dwfnamecommsdict
489   global newcommsdict
490
491   if len(linetoks) < 1:
492     return linetoks
493   tnum = firstnonblank(linetoks)
494   if tnum >= len(linetoks):
495     return linetoks
496   initialtok = linetoks[tnum]
497   itokstring=toknamestring(initialtok)
498   if itokstring == "\\expandafter\\def\\csname":
499     return linetoks
500   if in_lines_to_ignore(myfile,linenum) == "y":
501     return linetoks
502   if itokstring == "\\newcommand":
503     t1 = linetoks[tnum+1]
504     if not isbrace(t1,'{'):
505       print "Improper character in newcommand", myfile,linenum
506       sys.exit(1)
507     t2 = linetoks[tnum+2]
508     if toknamestring(t2) == "\\simplenametablerule":
509        add_lines_to_ignore(myfile,linenum,linenum+18)
510     if toknamestring(t2) != "\\newdwfnamecommands":
511        tm = tokmention(t2,myfile,linenum)
512        applytodict(newcommsdict,toknamestring(t2),tm)
513     #Be silent on newdwfnamecommands, it is normal.
514     #else:
515     #   print "newcommand on newdwfnamecommands ignored intentionally."
516     return linetoks
517   elif itokstring == "\\newdwfnamecommands":
518     t1 = linetoks[tnum+1]
519     if not isbrace(t1,'{'):
520        print "Improper character in newdwfnamecommands", myfile._name,linenum
521        sys.exit(1)
522     # The token name string will be DWsomething and we want
523     # The token to appear as \DWsomething as that is how references
524     # The usages determine what secondary actions are applied.
525     # are coded.
526     t2 = linetoks[tnum+2]
527     tkmod = fileio.dwtoken()
528     tkmod.insertid("\\" + toknamestring(t2))
529     tm = tokmention(tkmod,myfile,linenum)
530     applytodict(dwfnamecommsdict,toknamestring(tkmod),tm)
531     return linetoks
532   return linetoks
533
534
535 def delsuffix(n,suf):
536   slen = len(suf)
537   nlen = len(n)
538   lastcharnum = nlen - slen
539   outstring = n[0:lastcharnum]
540   return outstring
541 def deloptionalprefix(n,pref):
542   if not n.startswith(pref):
543     return n
544   plen = len(pref)
545   nlen = len(n)
546   outstring = n[plen:nlen]
547   return outstring
548
549 def printodderr(rawname,comname,myfile,linenum):
550   print "Error: this looks like a command is missing: ",rawname,"tested as",comname," in ",myfile._name," at ",linenum
551
552 def rawnameiscommand(t,suff):
553    if not t.startswith("\\DW"):
554      return ""
555    if not t.endswith(suff):
556      return ""
557    commandname = delsuffix(t,suff)
558    return commandname
559
560 # Delete any leading backslash.
561 # Prefix the result with chap:
562 def makelinkname(t):
563    s = deloptionalprefix(t,"\\");
564    s2 = "chap:" + s
565    return s2;
566    
567
568 # Assumes all new commands known already.
569 # This deals with targets and links (various flavors).
570 def transfunc2(linetoks,myfile,linenum):
571   global newcommsdict
572   global dwfnamecommsdict
573   global newcommsdict
574
575   # Link naming target
576   global linkhyperdict
577   # TARG suffix
578   global targhyperdict
579   # INDX suffix
580   global indexdict
581   # NAME suffix
582   global namedict
583
584   if len(linetoks) < 1:
585     return linetoks
586   if in_lines_to_ignore(myfile,linenum) == "y":
587     return linetoks
588   initialtok = linetoks[0]
589   itokstring=toknamestring(initialtok)
590   # Skip all the newcommand stuff.
591   if itokstring == "\\newcommand":
592     return linetoks
593   elif itokstring == '\\newdwfnamecommands':
594     return linetoks
595
596   # Now deal with a regular line.
597
598   tnumin = 0
599   changes = 0
600   lasttoknum = len(linetoks) -1
601   for x in linetoks:
602     if int(tnumin) > int(lasttoknum):
603       break
604     t = linetoks[tnumin]
605     if t._class != "id":
606       tnumin = tnumin + 1
607       continue
608     rawname = toknamestring(t)
609     commandname=""
610     #rawnameiscommand(rawname,"",basecommand)
611     if rawname == "\\expandafter\\def\\csname":
612       return linetoks
613     if rawname == "\\begin":
614       tnumcount = processbegin(linetoks,tnumin,myfile,linenum);
615       tnumin = tnumin + tnumcount
616       continue
617     if dwfnamecommsdict.has_key(rawname):
618       # We know this one. 
619       # It is a default case name reference.
620       # index the DWname
621       # Link is to chap:DWname
622       tm = tokmention(t,myfile,linenum)
623       linkname = makelinkname(rawname)
624       indxname = deloptionalprefix(commandname,"\\")
625       applytodict(indexdict,indxname,tm);
626
627       applytodict(linkhyperdict,linkname,tm);
628       tnumin = tnumin + 1
629       continue
630     if  newcommsdict.has_key(rawname):
631       # We know this one. We have to see what it is
632       # To decide what to do.
633       # some DWOPbreg*  DWOPreg*   and MDfive are special.
634       # A variety of other such defined commands are irrelevant to us here.
635
636       tnumcount = 1
637       if rawname == "\\livetarg":
638         tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"n")
639       elif rawname == "\\livetargi":
640         tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
641       elif rawname == "\\livelink":
642         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
643       elif rawname == "\\livelinki":
644         tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"n")
645       #elif rawname == "\\label":
646       #  tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
647       elif rawname == "\\refersec":
648         # does \ref
649         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
650       elif rawname == "\\referfol":
651         # does \vref from varioref package
652         tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
653       elif rawname == "\\index":
654         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
655       elif rawname == "\\addtoindex":
656         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
657       elif rawname == "\\addtoindexx":
658         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
659       elif rawname == "\\addttindex":
660         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
661       elif rawname == "\\addttindexx":
662         tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
663       elif rawname == "\\DWOPbregtwo":
664         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
665       elif rawname == "\\DWOPbregthree":
666         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
667       elif rawname == "\\DWOPbregfour":
668         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
669       elif rawname == "\\DWOPbregfive":
670         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
671       elif rawname == "\\DWOPbregeleven":
672         tnumcount = hyperlinkname("chap:DWOPbregn",tnumin,myfile,linenum)
673       elif rawname == "\\MDfive":
674         tnumcount = hyperlinkname("def:MDfive",tnumin,myfile,linenum)
675       else:
676         fake = ""
677         # If missing anything important, perhaps turn ths on.
678         #print "Error not handled: %s in file %s line %d" %(rawname,myfile._name,linenum)
679       tnumin = tnumin + tnumcount
680       continue
681     # Suffixes are LINK TARG INDX MARK NAME
682     commandname =rawnameiscommand(rawname,"LINK")
683     if len(commandname) > 0:
684       # index the DWname
685       # Link is to chap:DWname
686       if dwfnamecommsdict.has_key(commandname):
687         tm = tokmention(t,myfile,linenum)
688         linkname = makelinkname(commandname)
689         indxname = deloptionalprefix(commandname,"\\")
690         applytodict(linkhyperdict,linkname,tm)
691         applytodict(indexdict,indxname,tm);
692       else:
693         printodderr(rawname,commandname,myfile,linenum)
694       tnumin = tnumin + 1
695       continue
696     commandname =rawnameiscommand(rawname,"TARG")
697     if len(commandname) > 0:
698       # index DWname
699       # Set chap:DWname as having target defined
700       if dwfnamecommsdict.has_key(commandname):
701         tm = tokmention(t,myfile,linenum)
702         targname = makelinkname(commandname)
703         indxname = deloptionalprefix(commandname,"\\")
704         applytodict(targhyperdict,targname,tm)
705         applytodict(indexdict,indxname,tm);
706       else:
707         printodderr(rawname,commandname,myfile,linenum)
708       tnumin = tnumin + 1
709       continue
710     commandname =rawnameiscommand(rawname,"INDX")
711     if len(commandname) > 0:
712       # Index DWname
713       if dwfnamecommsdict.has_key(commandname):
714         tm = tokmention(t,myfile,linenum)
715         indexname = deloptionalprefix(commandname,"\\")
716         applytodict(indexdict,indexname,tm)
717       else:
718         printodderr(rawname,commandname,myfile,linenum)
719       tnumin = tnumin + 1
720       continue
721     commandname =rawnameiscommand(rawname,"MARK")
722     if len(commandname) > 0:
723       # set chap:DWname as target defined
724       # index DWname
725       if dwfnamecommsdict.has_key(commandname):
726         tm = tokmention(t,myfile,linenum)
727         targname = makelinkname(commandname)
728         applytodict(targhyperdict,targname,tm)
729         indexname = deloptionalprefix(commandname,"\\")
730         applytodict(indexdict,indexname,tm)
731       else:
732         printodderr(rawname,commandname,myfile,linenum)
733       tnumin = tnumin + 1
734       continue
735     commandname =rawnameiscommand(rawname,"NAME")
736     if len(commandname) > 0:
737       # No actions with NAME (but put in namedict anyway).
738       if dwfnamecommsdict.has_key(commandname):
739         tm = tokmention(t,myfile,linenum)
740         applytodict(namedict,commandname,tm)
741       else:
742         printodderr(rawname,commandname,myfile,linenum)
743       tnumin = tnumin + 1
744       continue
745     if rawname == "\\label":
746       # This is a builtin, not our newcommand.
747       tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
748       tnumin = tnumin + tnumcount
749       continue
750     if rawname == "\\hypertarget":
751       # This is a builtin, not our newcommand.
752       tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum,"y")
753       tnumin = tnumin + tnumcount
754       continue
755     if rawname == "\\hyperlink":
756       # This is a builtin, not our newcommand.
757       tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum,"y")
758       tnumin = tnumin + tnumcount
759       continue
760     # Some random data or text here.
761
762     tnumin = tnumin + 1
763     # We don't know what this is. Probably ok?
764     # End of for loop.
765   return linetoks
766
767 def sort_tokmlist(mylist):
768   aux = [ (''.join(x._token._tex),x) for x in mylist ]
769   aux.sort()
770   return[ (x[1]) for x in aux]
771
772 def printtokmention(ct,v):
773   n = v._token
774   name =  toknamestring(n)
775   f = v._file
776   l = v._line
777   print "    [%2d] %s in file %s line %d" %(ct,name,f._name,l)
778
779 # Are the lines close? If so 
780 def closetogether(l1,l2):
781   d = abs(l1 - l2)
782   if d < 4:
783     return "y"
784   return "n"
785
786 def shouldprintalldups(v):
787   if len(v) != 2:
788      return "y"
789   if v[0]._file != v[1]._file:
790      return "y"
791   if closetogether(v[0]._line,v[1]._line) == "y":
792         # Are the lines near one another? 
793         # If so a harmless duplication
794     return "n"
795   return "y"
796
797 def printtoomany(name,k,vlist):
798   print "Duplicate in %s: %s:" %(name,k)
799   ct = 0
800   for v in vlist:
801     printtokmention(ct,v)
802     ct = ct + 1
803
804 def checkduplicatetargs(dname,d):
805   targs = d.items()
806   for vi in targs:
807     (k,v) = vi
808     if len(v) > 1:
809       if shouldprintalldups(v) == "y":
810         printtoomany(dname,k,v)
811
812 def checkmissingtarg(name,targ,refs):
813   n = 0
814   rlist = refs.items()
815   for r in rlist:
816     (k,v) = r
817     if targ.has_key(k) == 0:
818        n = n + 1
819        print "Missing target from " + name + ":",k
820   if n == 0: print "No missing targets from " + name +"!"
821
822 def checkmissingref(name,targ,refs):
823   n = 0
824   rlist = targ.items()
825   for r in rlist:
826     (k,v) = r
827     if refs.has_key(k) == 0:
828        n = n + 1
829        print "Unused target from " + name + ":",k
830   if n == 0: print "No unused targets from " + name + "..."
831
832 def print_stats():
833   global dwfnamecommsdict
834   global newcommsdict
835   global targhyperdict
836   global linkhyperdict
837   global indexdict
838   global labeldict
839   global labelrefdict
840   
841   checkduplicatetargs("newdwfname commands",dwfnamecommsdict)
842   checkduplicatetargs("commands",newcommsdict)
843   checkduplicatetargs("hypertargets",targhyperdict)
844   checkduplicatetargs("labels",labeldict)
845
846   # Unused targets are nice to know
847   checkmissingref("hyperlinks",targhyperdict,linkhyperdict)
848   checkmissingref("labels",labeldict,labelrefdict)
849
850   # Missing targets are a problem to be FIXED
851   checkmissingtarg("hyperlinks",targhyperdict,linkhyperdict)
852   checkmissingtarg("labels",labeldict,labelrefdict)
853
854 # Perhaps these should be controlled by
855 # the command line.
856 debug   = "n"
857
858 def buildfilepaths(files,basetarg):
859   outlist = []
860   prefix = ""
861   for f in files:
862     prefix = ""
863     if len(basetarg) > 0:
864       prefix = basetarg
865     else:
866       prefix = os.path.join("..","latexdoc","")
867     outlist += [prefix + f]
868   return outlist
869 def read_all_args():
870   filelist1 = []
871   filelist2 = []
872   baselist1 = []
873   baselist2 = []
874   basetarg = ""
875   fileio.setkeepordeletecomments("d")
876   if debug == "y":
877     baselist1 = ["testrefer.tex"]
878     baselist2 = ["testrefer.tex"]
879     basetarg = "./"
880   else:
881     baselist1 = ["dwarfnamecmds.tex",
882               "dwarf5.tex",
883               "generaldescription.tex"]
884
885     baselist2 = ["dwarf5.tex",
886               "attributesbytag.tex",
887               "changesummary.tex",
888               "compression.tex",
889               "copyright.tex",
890               "dataobject.tex",
891               "datarepresentation.tex",
892               "debugsectionrelationships.tex",
893               "encodingdecoding.tex",
894               "examples.tex",
895               "foreword.tex",
896               "generaldescription.tex",
897               "gnulicense.tex",
898               "introduction.tex",
899               "otherdebugginginformation.tex",
900               "programscope.tex",
901               "sectionversionnumbers.tex",
902               "splitobjects.tex",
903               "typeentries.tex"]
904   filelist1 = buildfilepaths(baselist1,basetarg)
905   filelist2 = buildfilepaths(baselist2,basetarg)
906
907   if (len(filelist1) < 1) or (len(filelist2) < 1):
908     print >> sys.stderr , "No files specified to refer.py, internal error."
909     sys.exit(1)
910   # Pickup all the newcommand instances.
911   dwf = fileio.readFilelist(filelist1)
912   dwf.dwtransformline(transfunc1)
913
914   # Now find all the uses.
915   dwf2 = fileio.readFilelist(filelist2)
916   dwf2.dwtransformline(transfunc2)
917   print_stats()
918
919 if __name__ == '__main__':
920   read_all_args()
921