// myTok - returns next word from input stream, discarding
// numbers and punctuation, and converting to lower case.

include 'stringTo'

global class tokenizer
    {
    file f;		// Associated file
    string lineBuf = "";// Current line in file
    int line = 0;	// Line number in file
    int column = 0;	// Column in line
    string reuse = nil;	// If set to TRUE then reuse token
    to get() into string s
        {
	if (self.reuse)
	    {
	    s = self.reuse;
	    self.reuse = nil;
	    }
	else
	    {
	    int pos=self.column, string word;
	    if (self.f)
		{
		for (;;)
		    {
		    for (;;)
		        {
			(word,pos) = lineBuf.nextToken(pos);
			if (!word)
			    break;
			self.column = pos;
			s = word;
			return;
			}
		    self.line += 1;
		    self.lineBuf = self.f.readLine();
		    pos = 0;
		    if (!self.lineBuf)
			{
			self.f = nil;
			return;
			}
		    }
		}
	    }
	}
    to unget(string s)
        {
	self.reuse = s;
	}
    to errAt(string message)
        {
	punt("Line " + self.line + " " + self.f.name + ". " + message);
	}
    }

global to tokenizerNew(string fileName) into tokenizer tkz
{
tkz = ();
tkz.f = fileOpen(fileName, "r");
}