TTokenClass = enum 
  gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber, 
  gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit, 
  gtLongStringLit, gtCharLit, gtEscapeSequence, 
  gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression, 
  gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler, gtPreprocessor, 
  gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel, gtReference, gtOther

TGeneralTokenizer = object of TObject
  kind*: TTokenClass
  start*, length*: int
  buf: cstring
  pos: int
  state: TTokenClass

TSourceLanguage = enum 
  langNone, langNimrod, langCpp, langCsharp, langC, langJava

sourceLanguageToStr: array[TSourceLanguage, string] = ["none", "Nimrod", "C++", 
    "C#", "C", "Java"]

tokenClassToStr: array[TTokenClass, string] = ["Eof", "None", "Whitespace", 
    "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber", 
    "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit", 
    "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment", 
    "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData", 
    "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink", 
    "Label", "Reference", "Other"]

proc getSourceLanguage(name: string): TSourceLanguage {.raises: [], tags: [].}

proc initGeneralTokenizer(g: var TGeneralTokenizer; buf: cstring) {.raises: [], 
    tags: [].}

proc initGeneralTokenizer(g: var TGeneralTokenizer; buf: string) {.raises: [], 
    tags: [].}

proc deinitGeneralTokenizer(g: var TGeneralTokenizer) {.raises: [], tags: [].}

proc getNextToken(g: var TGeneralTokenizer; lang: TSourceLanguage) {.raises: [], 
    tags: [].}