package com.dbxml.db.core.indexer;

/*
 * dbXML - Native XML Database
 * Copyright (C) 1999-2004  The dbXML Group, L.L.C.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 *
 * $Id: IndexManager.java,v 1.10 2004/06/04 19:53:30 bradford Exp $
 */

import com.dbxml.util.*;
import java.util.*;

import com.dbxml.db.core.ClassResolver;
import com.dbxml.db.core.Collection;
import com.dbxml.db.core.Container;
import com.dbxml.db.core.DBException;
import com.dbxml.db.core.FaultCodes;
import com.dbxml.db.core.data.ContainerSet;
import com.dbxml.db.core.data.Key;
import com.dbxml.db.core.security.Access;
import com.dbxml.db.core.security.SecurityException;
import com.dbxml.db.core.security.User;
import com.dbxml.db.core.security.UserStack;
import com.dbxml.db.core.transaction.Transaction;
import com.dbxml.xml.QName;
import com.dbxml.xml.SymbolTable;
import com.dbxml.xml.dtsm.Constants;
import com.dbxml.xml.dtsm.DTSMException;
import com.dbxml.xml.dtsm.DocumentTable;
import com.dbxml.xml.dtsm.DocumentTableEntry;
import com.dbxml.xml.dtsm.PullProcessor;
import java.io.IOException;
import com.dbxml.xml.dtsm.Producer;
import com.dbxml.xml.dtsm.Consumer;

/**
 * IndexManager is a class that manages Indexes.  Good description, eh?
 * I should win a Pulitzer Prize for that one.
 */

public final class IndexManager extends SimpleConfigurable {
   private static final String[] EmptyStrings = new String[0];
   private static final IndexerInfo[] EmptyIndexerInfo = new IndexerInfo[0];

   private static final int AUTOINDEXER_WAIT = 1000;

   private static final String INDEX = "index";
   private static final String NAME = "name";
   private static final String CLASS = "class";

   private static final int STATUS_READY = 0;
   private static final int STATUS_BUILDING = 1;
   private static final int STATUS_DELETED = 2;

   private static final int ACTION_CREATE = 0;
   private static final int ACTION_UPDATE = 1;
   private static final int ACTION_DELETE = 2;

   private static AutoIndexer autoIndexer;
   private static User magicUser;
   private static UserStack userStack;

   private Map patternMap = Collections.synchronizedMap(new HashMap());   // IndexPattern to IndexerInfo
   private Map indexes = Collections.synchronizedMap(new TreeMap());      // String to IndexerInfo
   private Map bestIndexers = Collections.synchronizedMap(new TreeMap()); // String to SoftRefTab of IndexPattern to Indexer

   private List newIndexers = Collections.synchronizedList(new ArrayList()); // of IndexerInfo
   private IndexerInfo[] idxList = EmptyIndexerInfo;

   private Collection collection;
   private SymbolTable symbols;

   public IndexManager(Collection collection) {
      this.collection = collection;
      try {
         symbols = collection.getSymbols();
      }
      catch ( Exception e ) {
         e.printStackTrace(System.err);
      }
   }

   public void setConfig(Configuration config) throws dbXMLException {
      super.setConfig(config);

      try {
         userStack.pushCurrentUser(magicUser);

         config.processChildren(INDEX,
            new ConfigurationCallback() {
               public void process(Configuration cfg) {
                  String className = cfg.getAttribute(CLASS);
                  try {
                     register(ClassResolver.get(className), cfg);
                  }
                  catch ( Exception e ) {
                     e.printStackTrace(System.err);
                  }
               }
            });

         autoIndexer.checkNewIndexers();
      }
      catch ( Exception e ) {
         System.err.println(e.getMessage());
      }
      finally {
         userStack.popCurrentUser();
      }
   }

   public static void setMagicUser(User magicUser) throws SecurityException {
      if ( IndexManager.magicUser == null ) {
         IndexManager.magicUser = magicUser;
         if ( userStack != null )
            autoIndexer = new AutoIndexer();
      }
      else
         throw new SecurityException(FaultCodes.SEC_INVALID_ACCESS, "Can't overwrite Magic User");
   }

   public static void setUserStack(UserStack userStack) throws SecurityException {
      if ( IndexManager.userStack == null ) {
         IndexManager.userStack = userStack;
         if ( magicUser != null )
            autoIndexer = new AutoIndexer();
      }
      else
         throw new SecurityException(FaultCodes.SEC_INVALID_ACCESS, "Can't overwrite User Stack");
   }

   private void checkAccess(int permissions) throws SecurityException {
      collection.getDatabase().getSecurityManager().access(collection.getCanonicalName(), permissions);
   }

   /**
    * list returns a list of the Indexers that this IndexerManager has
    * registered.
    *
    * @return An array containing the Indexer names
    */
   public String[] list() {
      return (String[])indexes.keySet().toArray(EmptyStrings);
   }

   /**
    * drop physically removes the specified Indexer and any
    * associated system resources that the Indexer uses.
    *
    * @param name The Indexer to drop
    * @return Whether or not the Indexer was dropped
    */
   public boolean drop(final String name) throws DBException {
      checkAccess(Access.CREATE);

      IndexerInfo info = (IndexerInfo)indexes.get(name);
      if ( info == null )
         throw new IndexerException(FaultCodes.IDX_INDEX_NOT_FOUND, "Index '"+name+"' does not exist");

      info.status = STATUS_DELETED;

      unregister(name);
      config.processChildren(INDEX,
         new ConfigurationCallback() {
            public void process(Configuration cfg) {
               try {
                  if ( cfg.getAttribute(NAME).equals(name) )
                     cfg.delete();
               }
               catch ( Exception e ) {
                  e.printStackTrace(System.err);
               }
            }
         });

      boolean res = false;
      try {
         res = info.indexer.drop();
      }
      catch ( Exception e ) {
         e.printStackTrace(System.err);
      }
      collection.getDatabase().flushConfig();
      return res;
   }

   /**
    * create creates a new Indexer object and any associated
    * system resources that the Indexer will need.
    *
    * @param cfg The Indexer's configuration
    * @return The Indexer that was created
    */
   public Indexer create(Configuration cfg) throws DBException {
      checkAccess(Access.CREATE);

      String className = cfg.getAttribute(CLASS);
      try {
         // Check for duplicates
         String name = cfg.getAttribute(NAME);
         Configuration[] cfgs = config.getChildren();
         for ( int i = 0; i < cfgs.length; i++ )
            if ( cfgs[i].getAttribute(NAME).equals(name) )
               throw new DuplicateIndexException("Duplicate Index '" + name + "'");

         Class c = ClassResolver.get(className);
         Indexer idx = register(c, cfg);

         config.add(cfg);

         collection.getDatabase().flushConfig();

         autoIndexer.checkNewIndexers();
         return idx;
      }
      catch ( DBException d ) {
         throw d;
      }
      catch ( Exception e ) {
         throw new CannotCreateException("Cannot create Index", e);
      }
   }

   /**
    * closeAll closes all opened Indexers.  This method is called by
    * the containing Collection when it is being disposed of.
    */
   public void closeAll() {
      Iterator iter = indexes.values().iterator();
      while ( iter.hasNext() ) {
         IndexerInfo info = (IndexerInfo)iter.next();
         Indexer idx = info.indexer;

         try {
            if ( idx.isOpened() )
               idx.close();
         }
         catch ( Exception e ) {
            e.printStackTrace(System.err);
         }
      }
   }

   private Indexer register(Class c, Configuration cfg) throws DBException {
      String name = null;
      Transaction tx = new Transaction();

      try {
         name =  cfg.getAttribute(NAME);

         Indexer idx = (Indexer)c.newInstance();
         initialize(idx, cfg);
         name = idx.getName();

         if ( name == null || name.trim().equals("") )
            throw new CannotCreateException("No name specified");

         IndexPattern pattern = new IndexPattern(symbols, idx.getPattern(), null, true);
         String style = idx.getIndexStyle();
         IndexerInfo info = new IndexerInfo(name, style, pattern, idx);

         boolean newIndex = false;
         if ( !idx.exists() ) {
            idx.create();
            newIndex = true;
         }
         idx.open();

         if ( newIndex ) {
            info.status = STATUS_BUILDING;
            newIndexers.add(info);
            autoIndexer.addIndexManager(this);
         }
         else
            info.status = STATUS_READY;

         indexes.put(name, info);
         patternMap.put(pattern, info);

         Map tbl = (Map)bestIndexers.get(style);
         if ( tbl == null ) {
            tbl = new SoftHashMap();
            bestIndexers.put(style, tbl);
         }
         tbl.clear();
         idxList = (IndexerInfo[])indexes.values().toArray(EmptyIndexerInfo);

         collection.flushSymbolTable();

         return idx;
      }
      catch ( DBException d ) {
         tx.cancel();
         throw d;
      }
      catch ( Exception e ) {
          tx.cancel();
         throw new CannotCreateException("Cannot create Index '" + name + "'", e);
      }
      finally {
         if ( tx.getStatus() == Transaction.ACTIVE )
            tx.commit();
      }
   }

   private void unregister(String name) {
      IndexerInfo idx = (IndexerInfo)indexes.remove(name);
      String style = idx.style;
      patternMap.remove(idx.pattern);

      Map tbl = (Map)bestIndexers.get(style);
      if ( tbl == null ) {
         tbl = new SoftHashMap();
         bestIndexers.put(style, tbl);
      }
      tbl.clear();

      idxList = (IndexerInfo[])indexes.values().toArray(EmptyIndexerInfo);
   }

   private void initialize(Indexer idx, Configuration cfg) throws dbXMLException {
      idx.setCollection(collection);
      idx.setConfig(cfg);
   }

   private void populateNewIndexers() throws DBException {
      if ( newIndexers.size() > 0 ) {
         Transaction tx = new Transaction();
         try {
            IndexerInfo[] list = (IndexerInfo[])newIndexers.toArray(EmptyIndexerInfo);
            newIndexers.clear();

            ContainerSet docs = collection.getContainerSet(tx);
            while ( docs.hasMoreContainers() ) {
               Container c = docs.getNextContainer();
               if ( c != null ) {
                  try {
                     new IndexProducer(tx, c.getKey(), c.getDocument(), ACTION_CREATE, list);
                  }
                  catch ( IndexCancelException e ) {
                     break;
                  }
                  catch ( Exception e ) {
                     e.printStackTrace(System.err);
                  }
               }
            }

            for ( int i = 0; i < list.length; i++ ) {
               try {
                  if ( list[i].status != STATUS_DELETED ) {
                     list[i].indexer.flush(tx);
                     list[i].status = STATUS_READY;
                  }
               }
               catch ( Exception e ) {
               }
            }
         }
         catch ( DBException e ) {
            tx.cancel();
            throw e;
         }
         finally {
            if ( tx.getStatus() == Transaction.ACTIVE )
               tx.commit();
         }
      }
   }

   /**
    * get retrieves an Indexer by name.
    *
    * @param name The Indexer name
    * @return The Indexer
    */
   public Indexer get(String name) {
      IndexerInfo info = (IndexerInfo)indexes.get(name);
      if ( info != null )
         return info.indexer;
      else
         return null;
   }

   /**
    * getBestIndexer retrieves the best Indexer to use for the specified
    * IndexPattern.
    *
    * @param style The Indexer Style (ex: Node, Value)
    * @param pattern The IndexPattern to use
    * @return The best Indexer (or null)
    */
   public Indexer getBestIndexer(String style, IndexPattern pattern) {
      Map tbl = (Map)bestIndexers.get(style);
      if ( tbl == null ) {
         tbl = new SoftHashMap();
         bestIndexers.put(style, tbl);
      }
      Indexer idx = (Indexer)tbl.get(pattern);
      if ( idx == null ) {
         int highScore = 0;
         Iterator enum = indexes.values().iterator();
         while ( enum.hasNext() ) {
            IndexerInfo info = (IndexerInfo)enum.next();
            if ( info.status != STATUS_READY
               || !info.indexer.getIndexStyle().equals(style) )
               continue;
            int score = pattern.getMatchLevel(info.pattern);
            if ( score > highScore ) {
               idx = info.indexer;
               highScore = score;
            }
         }
         tbl.put(pattern, idx);
      }
      return idx;
   }

   public void addDocument(Transaction tx, Key key, DocumentTable doc) {
      if ( idxList.length > 0 ) {
         try {
            IndexerInfo[] list = new IndexerInfo[idxList.length];
            System.arraycopy(idxList, 0, list, 0, idxList.length);
            new IndexProducer(tx, key, doc, ACTION_CREATE, list);
            for ( int i = 0; i < list.length; i++ ) {
               try {
                  if ( list[i].status != STATUS_DELETED )
                     list[i].indexer.flush(tx);
               }
               catch ( Exception e ) {
               }
            }
         }
         catch ( IndexCancelException e ) {
            // Doesn't Matter
         }
      }
   }

   public void removeDocument(Transaction tx, Key key, DocumentTable doc) {
      if ( idxList.length > 0 ) {
         try {
            IndexerInfo[] list = new IndexerInfo[idxList.length];
            System.arraycopy(idxList, 0, list, 0, idxList.length);
            new IndexProducer(tx, key, doc, ACTION_DELETE, list);
            for ( int i = 0; i < list.length; i++ ) {
               try {
                  if ( list[i].status != STATUS_DELETED )
                     list[i].indexer.flush(tx);
               }
               catch ( Exception e ) {
               }
            }
         }
         catch ( IndexCancelException e ) {
            // Doesn't Matter
         }
      }
   }


   /**
    * IndexerInfo
    */

   private class IndexerInfo {
      public String name;
      public String style;
      public IndexPattern pattern;
      public Indexer indexer;
      public int status;

      public IndexerInfo(String name, String style, IndexPattern pattern, Indexer indexer) {
         this.name = name;
         this.style = style;
         this.pattern = pattern;
         this.indexer = indexer;
      }
   }


   /**
    * DocumentHandler actually performs the work of adding and removing Indexer
    * entries.
    */

   private class IndexProducer extends Producer implements Consumer {
      private Stack stack = new Stack();
      private IndexerInfo[] list;

      private Transaction tx;
      private Key key;
      private int action;

      public StackInfo info; // Current State

      public IndexProducer(Transaction tx, Key key, DocumentTable table, int action, IndexerInfo[] list) throws IndexCancelException {
         this.tx = tx;
         this.list = list;
         this.key = key;
         this.action = action;

         setDocumentTable(table);

         try {
            process();
         }
         catch ( IndexCancelException e ) {
            throw e;
         }
         catch ( Exception e ) {
            e.printStackTrace(System.err);
         }
      }

      public void process(Consumer consumer) throws Exception {
         try {
            super.process(consumer);

            if ( action == ACTION_CREATE || action == ACTION_UPDATE )
               collection.flushSymbolTable();
         }
         catch ( Exception e ) {
            throw e;
         }
      }

      public void beginDocument(DocumentTableEntry entry) {
      }

      public void endDocument(DocumentTableEntry entry) {
      }

      public void beginElement(DocumentTableEntry entry) throws Exception {
         int symID = entry.getValID();
         QName qname = IndexProducer.this.symbols.getQName(symID);

         qname = SymbolTable.getNormalizedQName(qname);
         symID = IndexProducer.this.symbols.getSymbol(qname, true);
         pushStackInfo(symID, processor.getPos());

         while ( processor.peek().getTypeID() == Constants.OBJ_BEGIN_ATTRIBUTE )
            attribute(processor.next());
      }

      public void endElement(DocumentTableEntry entry) throws Exception {
         if ( info.sb != null )
            processEntry(new IndexPattern(IndexManager.this.symbols, info.symbolID), info.sb.toString(), info.pos);
         popStackInfo();
      }

      public void attribute(DocumentTableEntry entry) throws Exception {
         int symID = entry.getValID();
         QName qname = IndexProducer.this.symbols.getQName(symID);

         qname = SymbolTable.getNormalizedQName(qname);
         symID = IndexProducer.this.symbols.getSymbol(qname, true);

         StringBuffer sb = new StringBuffer();
         do {
            entry = processor.next();
            if ( entry.getTypeID() == Constants.OBJ_TEXT ) {
               int valID = entry.getValID();
               sb.append(UTF8.toCharArray(table.getValue(valID)));
            }
         }
         while ( entry.getTypeID() != Constants.OBJ_END_ATTRIBUTE );

         if ( sb.length() > 0 )
            processEntry(new IndexPattern(IndexManager.this.symbols, info.symbolID, symID), sb.toString(), info.pos);
      }

      public void comment(DocumentTableEntry entry) {
      }

      public void entityRef(DocumentTableEntry entry) {
      }

      public void procInst(DocumentTableEntry entry) throws Exception {
         do {
            entry = processor.next();
         }
         while ( entry.getTypeID() != Constants.OBJ_END_PROCINST );
      }

      public void text(DocumentTableEntry entry) throws Exception {
         int valID = entry.getValID();
         String val = table.getValue(valID).toString().trim();
         if ( info.sb == null )
            info.sb = new StringBuffer(val.length());
         else if ( info.sb.length() > 0 )
            info.sb.append(' ');
         info.sb.append(val);
      }

      public void cdata(DocumentTableEntry entry) throws Exception {
         int valID = entry.getValID();
         String val = table.getValue(valID).toString().trim();
         if ( info.sb == null )
            info.sb = new StringBuffer(val.length());
         else if ( info.sb.length() > 0 )
            info.sb.append(' ');
         info.sb.append(val);
      }

      public void docType(DocumentTableEntry entry) {
         do {
            entry = processor.next();
         }
         while ( entry.getTypeID() != Constants.OBJ_END_DOCTYPE );
      }

      public void entity(DocumentTableEntry entry) {
         do {
            entry = processor.next();
         }
         while ( entry.getTypeID() != Constants.OBJ_END_ENTITY );
      }

      public void notation(DocumentTableEntry entry) {
         do {
            entry = processor.next();
         }
         while ( entry.getTypeID() != Constants.OBJ_END_NOTATION );
      }

      private void pushStackInfo(int symID, int pos) {
         if ( info != null )
            stack.push(info);
         info = new StackInfo(symID, pos);
      }

      private void popStackInfo() {
         if ( !stack.empty() )
            info = (StackInfo)stack.pop();
         else
            info = null;
      }

      private void processEntry(IndexPattern pattern, String value, int pos) throws IndexCancelException {
         int delCount = 0;
         for ( int i = 0; i < list.length; i++ ) {
            IndexerInfo info = list[i];
            if ( info.status == STATUS_DELETED ) {
               delCount++;
               continue;
            }
            else if ( pattern.getMatchLevel(info.pattern) > 0 ) {
               try {
                  switch ( action ) {
                     case ACTION_CREATE:
                     case ACTION_UPDATE:
                        list[i].indexer.add(tx, value, key, pos, pattern.getElementID(), pattern.getAttributeID());
                        break;
                     case ACTION_DELETE:
                        list[i].indexer.remove(tx, value, key, pos, pattern.getElementID(), pattern.getAttributeID());
                        break;
                  }
               }
               catch ( Exception e ) {
                  e.printStackTrace(System.err);
               }
            }
         }
         if ( delCount == list.length )
            throw new IndexCancelException();
      }
   }


   /**
    * StackInfo
    */

   private class StackInfo {
      public int symbolID;
      public int pos = -1;
      public StringBuffer sb;

      public StackInfo(int symbolID, int pos) {
         this.symbolID = symbolID;
         this.pos = pos;
      }
   }


   /**
    * IndexCancelException
    */

   private class IndexCancelException extends Exception {
   }


   /**
    * AutoIndexer
    */

   private static class AutoIndexer extends Thread {
      Set mgrSet = new HashSet();

      public AutoIndexer() {
         setName("Automatic Indexer");
         start();
      }

      public void run() {
         userStack.pushCurrentUser(magicUser);
         while ( true ) {
            Set set;

            synchronized ( mgrSet ) {
               while ( mgrSet.isEmpty() ) {
                  try {
                     mgrSet.wait(AUTOINDEXER_WAIT);
                  }
                  catch ( Exception e ) {
                  }
               }

               set = new HashSet(mgrSet);
            }

            Iterator iter = set.iterator();
            while ( iter.hasNext() ) {
               IndexManager mgr = (IndexManager)iter.next();

               // Remove from the Original IndexManager Set
               synchronized ( mgrSet ) {
                  mgrSet.remove(mgr);
               }

               try {
                  mgr.populateNewIndexers();
               }
               catch ( Exception e ) {
                  e.printStackTrace(System.err);
               }
            }
         }
      }

      public void addIndexManager(IndexManager mgr) {
         synchronized ( mgrSet ) {
            mgrSet.add(mgr);
         }
      }

      public void checkNewIndexers() {
         synchronized ( mgrSet ) {
            mgrSet.notify();
         }
      }
   }
}



