/*
 * DefaultGazeteer.java
 *
 * Copyright (c) 2000-2001, The University of Sheffield.
 *
 * This file is part of GATE (see http://gate.ac.uk/), and is free
 * software, licenced under the GNU Library General Public License,
 * Version 2, June1991.
 *
 * A copy of this licence is included in the distribution in the file
 * licence.html, and is also available at http://gate.ac.uk/gate/licence.html.
 *
 * Valentin Tablan, 03/07/2000
 *
 * $Id: DefaultGazetteer.java,v 1.33 2001/11/12 15:04:28 valyt Exp $
 */

package gate.creole.gazetteer;

import java.io.*;
import java.util.*;
import java.net.*;

import gate.util.*;
import gate.creole.*;
import gate.event.*;
import gate.*;

/** This component is responsible for doing lists lookup. The implementaion is
 * based on finite state machines.
 * The phrases to be recognised should be listed in a set of files, one for
 * each type of occurences.
 * The gazeteer is build with the information from a file that contains the set
 * of lists (which are files as well) and the associated type for each list.
 * The file defining the set of lists should have the following syntax:
 * each list definition should be written on its own line and should contain:
 * <ol>
 * <li>the file name (required) </li>
 * <li>the major type (required) </li>
 * <li>the minor type (optional)</li>
 * <li>the language(s) (optional) </li>
 * </ol>
 * The elements of each definition are separated by &quot;:&quot;.
 * The following is an example of a valid definition: <br>
 * <code>personmale.lst:person:male:english</code>
 * Each list file named in the lists definition file is just a list containing
 * one entry per line.
 * When this gazetter will be run over some input text (a Gate document) it
 * will generate annotations of type Lookup having the attributes specified in
 * the definition file.
 */
public class DefaultGazetteer extends AbstractLanguageAnalyser
             implements ProcessingResource {

  /** Debug flag
   */
  private static final boolean DEBUG = false;

  /** Build a gazetter using the default lists from the agte resources
   * {@see init()}
   */
  public DefaultGazetteer(){
  }

  /** Does the actual loading and parsing of the lists. This method must be
   * called before the gazetteer can be used
   */
  public Resource init()throws ResourceInstantiationException{
    fsmStates = new HashSet();
    try{
      initialState = new FSMState(this);
      if(listsURL == null){
        throw new ResourceInstantiationException (
              "No URL provided for gazetteer creation!");
      }

      //find the number of lines
      Reader reader = new InputStreamReader(listsURL.openStream(), encoding);
      int linesCnt = 0;
      BufferedReader bReader = new BufferedReader(reader);
      String line = bReader.readLine();
      while (line != null) {
        linesCnt++;
        line = bReader.readLine();
      }
      bReader.close();

      //parse the file
      reader = new InputStreamReader(listsURL.openStream(), encoding);
      bReader = new BufferedReader(reader);
      line = bReader.readLine();
      ///String toParse = "";
      StringBuffer toParse = new StringBuffer(gate.Gate.STRINGBUFFER_SIZE);

      int lineIdx = 0;
      while (line != null) {
        if(line.endsWith("\\")) {
          ///toParse += line.substring(0,line.length()-1);
          toParse.append(line.substring(0,line.length()-1));
        } else {
          ///toParse += line;
          toParse.append(line);
          fireStatusChanged("Reading " + toParse.toString());
          fireProgressChanged(lineIdx * 100 / linesCnt);
          lineIdx ++;
          readList(toParse.toString(), true);
          ///toParse = "";
          toParse.delete(0,toParse.length());
        }
        line = bReader.readLine();
      }
      fireProcessFinished();
    }catch(IOException ioe){
      throw new ResourceInstantiationException(ioe);
    }catch(GazetteerException ge){
      throw new ResourceInstantiationException(ge);
    }
    return this;
  }


  /** Reads one lists (one file) of phrases
   *
   * @param listDesc the line from the definition file
   * @param add
   * @add if <b>true</b> will add the phrases found in the list to the ones
   *     recognised by this gazetter, if <b>false</b> the phrases found in the
   *     list will be removed from the list of phrases recognised by this
   *     gazetteer.
   */
  void readList(String listDesc, boolean add) throws FileNotFoundException,
                                        IOException,
                                        GazetteerException{
    String listName, majorType, minorType, languages;
    int firstColon = listDesc.indexOf(':');
    int secondColon = listDesc.indexOf(':', firstColon + 1);
    int thirdColon = listDesc.indexOf(':', secondColon + 1);
    if(firstColon == -1){
      throw new GazetteerException("Invalid list definition: " + listDesc);
    }
    listName = listDesc.substring(0, firstColon);

    if(secondColon == -1){
      majorType = listDesc.substring(firstColon + 1);
      minorType = null;
      languages = null;
    } else {
      majorType = listDesc.substring(firstColon + 1, secondColon);
      if(thirdColon == -1) {
        minorType = listDesc.substring(secondColon + 1);
        languages = null;
      } else {
        minorType = listDesc.substring(secondColon + 1, thirdColon);
        languages = listDesc.substring(thirdColon + 1);
      }
    }
    BufferedReader listReader;

    listReader = new BufferedReader(new InputStreamReader(
                            (new URL(listsURL, listName)).openStream(), encoding));

    Lookup lookup = new Lookup(majorType, minorType, languages);
    String line = listReader.readLine();
    while(null != line){
      if(add)addLookup(line, lookup);
      else removeLookup(line, lookup);
      line = listReader.readLine();
    }
  } // void readList(String listDesc)

  /** Adds one phrase to the list of phrases recognised by this gazetteer
   *
   * @param text the phrase to be added
   * @param lookup the description of the annotation to be added when this
   *     phrase is recognised
   */
// >>> DAM, was
/*
  public void addLookup(String text, Lookup lookup) {
    Character currentChar;
    FSMState currentState = initialState;
    FSMState nextState;
    Lookup oldLookup;
    boolean isSpace;

    for(int i = 0; i< text.length(); i++) {
      isSpace = Character.isWhitespace(text.charAt(i));
      if(isSpace) currentChar = new Character(' ');
      else currentChar = (caseSensitive.booleanValue()) ?
                          new Character(text.charAt(i)) :
                          new Character(Character.toUpperCase(text.charAt(i))) ;
      nextState = currentState.next(currentChar);
      if(nextState == null){
        nextState = new FSMState(this);
        currentState.put(currentChar, nextState);
        if(isSpace) nextState.put(new Character(' '),nextState);
      }
      currentState = nextState;
    } //for(int i = 0; i< text.length(); i++)

    currentState.addLookup(lookup);
    //Out.println(text + "|" + lookup.majorType + "|" + lookup.minorType);

  } // addLookup
*/
// >>> DAM: TransArray optimization
  public void addLookup(String text, Lookup lookup) {
    char currentChar;
    FSMState currentState = initialState;
    FSMState nextState;
    Lookup oldLookup;
    boolean isSpace;

    for(int i = 0; i< text.length(); i++) {
        currentChar = text.charAt(i);
        isSpace = Character.isWhitespace(currentChar);
        if(isSpace) currentChar = ' ';
        else currentChar = (caseSensitive.booleanValue()) ?
                          currentChar :
                          Character.toUpperCase(currentChar) ;
      nextState = currentState.next(currentChar);
      if(nextState == null){
        nextState = new FSMState(this);
        currentState.put(currentChar, nextState);
        if(isSpace) nextState.put(' ',nextState);
      }
      currentState = nextState;
    } //for(int i = 0; i< text.length(); i++)

    currentState.addLookup(lookup);
    //Out.println(text + "|" + lookup.majorType + "|" + lookup.minorType);

  } // addLookup
// >>> DAM, end

  /** Removes one phrase to the list of phrases recognised by this gazetteer
   *
   * @param text the phrase to be removed
   * @param lookup the description of the annotation associated to this phrase
   */
// >>> DAM, was
/*
  public void removeLookup(String text, Lookup lookup) {
    Character currentChar;
    FSMState currentState = initialState;
    FSMState nextState;
    Lookup oldLookup;
    boolean isSpace;

    for(int i = 0; i< text.length(); i++) {
      isSpace = Character.isWhitespace(text.charAt(i));
      if(isSpace) currentChar = new Character(' ');
      else currentChar = new Character(text.charAt(i));
      nextState = currentState.next(currentChar);
      if(nextState == null) return;//nothing to remove
      currentState = nextState;
    } //for(int i = 0; i< text.length(); i++)
    currentState.removeLookup(lookup);
  } // removeLookup
*/
// >>> DAM: TransArray optimization
  public void removeLookup(String text, Lookup lookup) {
    char currentChar;
    FSMState currentState = initialState;
    FSMState nextState;
    Lookup oldLookup;

    for(int i = 0; i< text.length(); i++) {
        currentChar = text.charAt(i);
        if(Character.isWhitespace(currentChar)) currentChar = ' ';
        nextState = currentState.next(currentChar);
        if(nextState == null) return;//nothing to remove
        currentState = nextState;
    } //for(int i = 0; i< text.length(); i++)
    currentState.removeLookup(lookup);
  } // removeLookup
// >>> DAM, end

  /** Returns a string representation of the deterministic FSM graph using
   * GML.
   */
  public String getFSMgml() {
    String res = "graph[ \ndirected 1\n";
    ///String nodes = "", edges = "";
    StringBuffer nodes = new StringBuffer(gate.Gate.STRINGBUFFER_SIZE),
                edges = new StringBuffer(gate.Gate.STRINGBUFFER_SIZE);
    Iterator fsmStatesIter = fsmStates.iterator();
    while (fsmStatesIter.hasNext()){
      FSMState currentState = (FSMState)fsmStatesIter.next();
      int stateIndex = currentState.getIndex();
      /*nodes += "node[ id " + stateIndex +
               " label \"" + stateIndex;
      */
      nodes.append("node[ id ");
      nodes.append(stateIndex);
      nodes.append(" label \"");
      nodes.append(stateIndex);

             if(currentState.isFinal()){
              ///nodes += ",F\\n" + currentState.getLookupSet();
              nodes.append(",F\\n");
              nodes.append(currentState.getLookupSet());
             }
             ///nodes +=  "\"  ]\n";
             nodes.append("\"  ]\n");
      //edges += currentState.getEdgesGML();
      edges.append(currentState.getEdgesGML());
    }
    res += nodes.toString() + edges.toString() + "]\n";
    return res;
  } // getFSMgml

  //no doc required: javadoc will copy it from the interface
  /**    */
  public FeatureMap getFeatures(){
    return features;
  } // getFeatures

  /**    */
  public void setFeatures(FeatureMap features){
    this.features = features;
  } // setFeatures



  /**
   * This method runs the gazetteer. It assumes that all the needed parameters
   * are set. If they are not, an exception will be fired.
   */
  public void execute() throws ExecutionException{
    interrupted = false;
    AnnotationSet annotationSet;
    //check the input
    if(document == null) {
      throw new ExecutionException(
        "No document to process!"
      );
    }

    if(annotationSetName == null ||
       annotationSetName.equals("")) annotationSet = document.getAnnotations();
    else annotationSet = document.getAnnotations(annotationSetName);

    fireStatusChanged("Doing lookup in " +
                           document.getSourceUrl().getFile() + "...");
    String content = document.getContent().toString();
    int length = content.length();
// >>> DAM, was
/*
    Character currentChar;
*/
// >>> DAM: TransArray optimization
    char currentChar;
// >>> DAM, end
    FSMState currentState = initialState;
    FSMState nextState;
    FSMState lastMatchingState = null;
    int matchedRegionEnd = 0;
    int matchedRegionStart = 0;
    int charIdx = 0;
    int oldCharIdx = 0;
    FeatureMap fm;
    Lookup currentLookup;

// >>> DAM, was
/*
    while(charIdx < length) {
      if(Character.isWhitespace(content.charAt(charIdx)))
        currentChar = new Character(' ');
      else currentChar = (caseSensitive.booleanValue()) ?
                         new Character(content.charAt(charIdx)) :
                         new Character(Character.toUpperCase(
                                       content.charAt(charIdx)));
*/
// >>> DAM: TransArray optimization
    while(charIdx < length) {
      currentChar = content.charAt(charIdx);
      if(Character.isWhitespace(currentChar)) currentChar = ' ';
      else currentChar = caseSensitive.booleanValue() ?
                          currentChar :
                          Character.toUpperCase(currentChar);
// >>> DAM, end
      nextState = currentState.next(currentChar);
      if(nextState == null) {
        //the matching stopped

        //if we had a successful match then act on it;
        if(lastMatchingState != null){
          //let's add the new annotation(s)
          Iterator lookupIter = lastMatchingState.getLookupSet().iterator();

          while(lookupIter.hasNext()) {
            currentLookup = (Lookup)lookupIter.next();
            fm = Factory.newFeatureMap();
            fm.put("majorType", currentLookup.majorType);
            if(null != currentLookup.minorType) {
              fm.put("minorType", currentLookup.minorType);
              if(null != currentLookup.languages)
                fm.put("language", currentLookup.languages);
            }
            try {
              annotationSet.add(new Long(matchedRegionStart),
                              new Long(matchedRegionEnd + 1),
                              "Lookup",
                              fm);
            } catch(InvalidOffsetException ioe) {
              throw new LuckyException(ioe.toString());
            }
          }//while(lookupIter.hasNext())
          lastMatchingState = null;
        }

        //reset the FSM
        charIdx = matchedRegionStart + 1;
        matchedRegionStart = charIdx;
        currentState = initialState;

      } else{//go on with the matching
        currentState = nextState;
        //if we have a successful state then store it
        if(currentState.isFinal() &&
           (matchedRegionStart == 0 ||
            !Character.isLetter(content.charAt(matchedRegionStart - 1))) &&
           (charIdx + 1 >= content.length()   ||
            !Character.isLetter(content.charAt(charIdx + 1)))
          ){
          matchedRegionEnd = charIdx;
          lastMatchingState = currentState;
        }
        charIdx ++;
        if(charIdx == content.length()){
          //we can't go on, use the last matching state and restart matching
          //from the next char
          if(lastMatchingState != null){
            //let's add the new annotation(s)
            Iterator lookupIter = lastMatchingState.getLookupSet().iterator();

            while(lookupIter.hasNext()) {
              currentLookup = (Lookup)lookupIter.next();
              fm = Factory.newFeatureMap();
              fm.put("majorType", currentLookup.majorType);
              if(null != currentLookup.minorType) {
                fm.put("minorType", currentLookup.minorType);
                if(null != currentLookup.languages)
                  fm.put("language", currentLookup.languages);
              }
              try {
                annotationSet.add(new Long(matchedRegionStart),
                                new Long(matchedRegionEnd + 1),
                                "Lookup",
                                fm);
              } catch(InvalidOffsetException ioe) {
                throw new LuckyException(ioe.toString());
              }
            }//while(lookupIter.hasNext())
            lastMatchingState = null;
          }

          //reset the FSM
          charIdx = matchedRegionStart + 1;
          matchedRegionStart = charIdx;
          currentState = initialState;
        }
      }
      if(charIdx - oldCharIdx > 256) {
        fireProgressChanged((100 * charIdx )/ length );
        oldCharIdx = charIdx;
        if(isInterrupted()) throw new ExecutionInterruptedException(
            "The execution of the " + getName() +
            " gazetteer has been abruptly interrupted!");
      }
    } // while(charIdx < length)

    if(lastMatchingState != null) {
      Iterator lookupIter = lastMatchingState.getLookupSet().iterator();
      while(lookupIter.hasNext()) {
        currentLookup = (Lookup)lookupIter.next();
        fm = Factory.newFeatureMap();
        fm.put("majorType", currentLookup.majorType);
        if(null != currentLookup.minorType)
          fm.put("minorType", currentLookup.minorType);
        try{
          annotationSet.add(new Long(matchedRegionStart),
                          new Long(matchedRegionEnd + 1),
                          "Lookup",
                          fm);
        } catch(InvalidOffsetException ioe) {
          throw new GateRuntimeException(ioe.toString());
        }
      }//while(lookupIter.hasNext())
    }
    fireProcessFinished();
    fireStatusChanged("Lookup complete!");
  } // execute


  /**
   * Sets the AnnotationSet that will be used at the next run for the newly
   * produced annotations.
   */
  public void setAnnotationSetName(String newAnnotationSetName) {
    annotationSetName = newAnnotationSetName;
  }


  /** The initial state of the FSM that backs this gazetteer
   */
  FSMState initialState;

  /** A set containing all the states of the FSM backing the gazetteer
   */
  Set fsmStates;

  protected FeatureMap features  = null;

  /** Used to store the annotation set currently being used for the newly
   * generated annotations
   */
  protected String annotationSetName;

  private String encoding = "UTF-8";

  /**
   * The value of this property is the URL that will be used for reading the
   * lists dtaht define this Gazetteer
   */
  private java.net.URL listsURL;

  /**
   * Should this gazetteer be case sensitive. The default value is true.
   */
  private Boolean caseSensitive = new Boolean(true);

  public void setEncoding(String newEncoding) {
    encoding = newEncoding;
  }
  public String getEncoding() {
    return encoding;
  }
  public void setListsURL(java.net.URL newListsURL) {
    listsURL = newListsURL;
  }
  public java.net.URL getListsURL() {
    return listsURL;
  }
  public void setCaseSensitive(Boolean newCaseSensitive) {
    caseSensitive = newCaseSensitive;
  }
  public Boolean getCaseSensitive() {
    return caseSensitive;
  }
  public String getAnnotationSetName() {
    return annotationSetName;
  }

} // DefaultGazetteer

// >>> DAM: TransArray optimization, new charMap implementation
interface Iter
{
    public boolean hasNext();
    public char next();
} // iter class

/**
 * class implementing the map using binary serach by char as key
 * to retrive the coresponding object.
 */
class charMap
{
    char[] itemsKeys = null;
    Object[] itemsObjs = null;

    /**
     * resize the containers by one leavaing empty elemant at position 'index'
     */
    void resize(int index)
    {
        int newsz = itemsKeys.length + 1;
        char[] tempKeys = new char[newsz];
        Object[] tempObjs = new Object[newsz];
        int i;
        for (i= 0; i < index; i++)
        {
            tempKeys[i] = itemsKeys[i];
            tempObjs[i] = itemsObjs[i];
        }
        for (i= index+1; i < newsz; i++)
        {
            tempKeys[i] = itemsKeys[i-1];
            tempObjs[i] = itemsObjs[i-1];
        }

        itemsKeys = tempKeys;
        itemsObjs = tempObjs;
    } // resize

/**
 * get the object from the map using the char key
 */
    Object get(char key)
    {
        if (itemsKeys == null) return null;
        int index = Arrays.binarySearch(itemsKeys, key);
        if (index<0)
            return null;
        return itemsObjs[index];
    }
/**
 * put the object into the char map using the chat as the key
 */
    Object put(char key, Object value)
    {
        if (itemsKeys == null)
        {
            itemsKeys = new char[1];
            itemsKeys[0] = key;
            itemsObjs = new Object[1];
            itemsObjs[0] = value;
            return value;
        }// if first time
        int index = Arrays.binarySearch(itemsKeys, key);
        if (index<0)
        {
            index = ~index;
            resize(index);
            itemsKeys[index] = key;
            itemsObjs[index] = value;
        }
        return itemsObjs[index];
    } // put
/**
 * the keys itereator
 * /
    public Iter iter()
    {
        return new Iter()
        {
            int counter = 0;
            public boolean hasNext() {return counter < itemsKeys.length;}
            public char next() { return itemsKeys[counter];}
        };
    } // iter()
 */

} // class charMap
// >>> DAM, end, new charMap instead MAP for transition function in the FSMState