svn commit: r1532273 - in /ofbiz/trunk/specialpurpose/lucene: servicedef/ src/org/ofbiz/content/search/

[jacopoc]

Author: jacopoc
Date: Tue Oct 15 10:13:11 2013
New Revision: 1532273

URL: http://svn.apache.org/r1532273
Log:
Refactored and simplified the old code to index Content record: it now leverages the new LuceneDocument interface and index handling facilities.

Modified:
    ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java
    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java
    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java

Modified: ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml (original)
+++ ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml Tue Oct 15 10:13:11 2013
@@ -27,8 +27,6 @@ under the License.
              location="org.ofbiz.content.search.SearchServices" invoke="indexTree">
         <description>Index content under publish point</description>
         <attribute mode="IN" name="contentId" optional="false" type="String"/>
-        <attribute mode="OUT" name="badIndexList" optional="true" type="List"/>
-        <attribute mode="OUT" name="goodIndexCount" optional="true" type="Integer"/>
     </service>
 
     <service name="indexProduct" engine="java" location="org.ofbiz.content.search.SearchServices" invoke="indexProduct">

Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java Tue Oct 15 10:13:11 2013
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import org.apache.lucene.index.Term;
 import org.ofbiz.base.util.Debug;
 import org.ofbiz.base.util.GeneralException;
 import org.ofbiz.base.util.StringUtil;
@@ -48,12 +49,29 @@ import org.apache.lucene.document.TextFi
  * ContentDocument Class
  */
 
-public class ContentDocument {
+public class ContentDocument implements LuceneDocument {
 
-    public static final String module = ContentDocument.class.getName();
+    private static final String module = ContentDocument.class.getName();
+    private final Term documentIdentifier;
+    private final LocalDispatcher dispatcher;
+    private final GenericValue content;
+
+    public ContentDocument(GenericValue content, LocalDispatcher dispatcher) {
+        this.content = content;
+        this.dispatcher = dispatcher;
+        this.documentIdentifier = new Term("contentId", content.getString("contentId"));
+    }
 
-    public static Document Document(GenericValue content, Map<String, Object> context, LocalDispatcher dispatcher) throws InterruptedException {
+    @Override
+    public String toString() {
+        return getDocumentIdentifier().toString();
+    }
 
+    public Term getDocumentIdentifier() {
+        return documentIdentifier;
+    }
+
+    public Document prepareDocument(Delegator delegator) {
         Document doc;
         // make a new, empty document
         doc = new Document();
@@ -82,26 +100,23 @@ public class ContentDocument {
             Field field = new StringField("site", ancestorString, Store.NO);
             doc.add(field);
         }
-        boolean retVal = indexDataResource(content, doc, context, dispatcher);
-        if (!retVal)
+        boolean retVal = indexDataResource(doc);
+        if (!retVal) {
             doc = null;
+        }
         return doc;
     }
 
-    public static boolean indexDataResource(GenericValue content, Document doc, Map<String, Object> context, LocalDispatcher dispatcher) {
+    private boolean indexDataResource(Document doc) {
         String contentId = content.getString("contentId");
         GenericValue dataResource;
         try {
             dataResource = content.getRelatedOne("DataResource", true);
         } catch (GenericEntityException e) {
             Debug.logError(e, module);
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(contentId + " - " + e.getMessage());
             return false;
         }
         if (dataResource == null) {
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(contentId + " - dataResource is null.");
             return false;
         }
         String mimeTypeId = dataResource.getString("mimeTypeId");
@@ -115,16 +130,12 @@ public class ContentDocument {
         }
         String text;
         try {
-            text = ContentWorker.renderContentAsText(dispatcher, content.getDelegator(), contentId, context, locale, mimeTypeId, true);
+            text = ContentWorker.renderContentAsText(dispatcher, content.getDelegator(), contentId, null, locale, mimeTypeId, true);
         } catch (GeneralException e) {
             Debug.logError(e, module);
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(contentId + " - " + e.getMessage());
             return false;
         } catch (IOException e2) {
             Debug.logError(e2, module);
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(contentId + " - " + e2.getMessage());
             return false;
         }
         if (UtilValidate.isNotEmpty(text)) {
@@ -136,8 +147,6 @@ public class ContentDocument {
             featureDataResourceList = content.getRelated("ProductFeatureDataResource", null, null, true);
         } catch (GenericEntityException e) {
             Debug.logError(e, module);
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(contentId + " - " + e.getMessage());
             return false;
         }
         List<String> featureList = new ArrayList<String>();

Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java Tue Oct 15 10:13:11 2013
@@ -21,15 +21,12 @@ package org.ofbiz.content.search;
 import java.lang.Object;
 import java.lang.String;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.servlet.http.HttpSession;
 
 import org.ofbiz.base.util.Debug;
-import org.ofbiz.base.util.StringUtil;
-import org.ofbiz.base.util.UtilGenerics;
 import org.ofbiz.base.util.UtilHttp;
 import org.ofbiz.base.util.UtilValidate;
 import org.ofbiz.entity.GenericValue;
@@ -64,16 +61,7 @@ public class SearchEvents {
             return "error";
         }
         String errMsg = ServiceUtil.getErrorMessage(result);
-        if (Debug.infoOn()) Debug.logInfo("errMsg:" + errMsg, module);
-        if (Debug.infoOn()) Debug.logInfo("result:" + result, module);
         if (UtilValidate.isEmpty(errMsg)) {
-            List<String> badIndexList = UtilGenerics.checkList(result.get("badIndexList"));
-            if (Debug.infoOn()) Debug.logInfo("badIndexList:" + badIndexList, module);
-            String badIndexMsg = StringUtil.join(badIndexList, "\n") + badIndexList.size() + " entities not indexed";
-            Integer goodIndexCount = (Integer)result.get("goodIndexCount");
-            String goodIndexMsg = goodIndexCount + " entities indexed.";
-            if (Debug.infoOn()) Debug.logInfo("goodIndexCount:" + goodIndexCount, module);
-            ServiceUtil.setMessages(request, badIndexMsg, goodIndexMsg, null);
             return "success";
         } else {
             ServiceUtil.setMessages(request, errMsg, null, null);

Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java Tue Oct 15 10:13:11 2013
@@ -20,9 +20,6 @@ package org.ofbiz.content.search;
 
 import java.lang.Object;
 import java.lang.String;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -49,30 +46,18 @@ public class SearchServices {
     public static final String resource = "ContentUiLabels";
 
     public static Map<String, Object> indexTree(DispatchContext dctx, Map<String, ? extends Object> context) {
-        Date start = new Date();
         LocalDispatcher dispatcher = dctx.getDispatcher();
         Delegator delegator = dctx.getDelegator();
         String siteId = (String) context.get("contentId");
         Locale locale = (Locale) context.get("locale");
-        Map<String, Object> envContext = new HashMap<String, Object>();
-
-        if (Debug.infoOn()) Debug.logInfo("in indexTree, siteId:" + siteId, module);
-        List<String> badIndexList = new ArrayList<String>();
-        envContext.put("badIndexList", badIndexList);
-        envContext.put("goodIndexCount", Integer.valueOf(0));
-
-        Map<String, Object> results;
         try {
-            results = SearchWorker.indexTree(dispatcher, delegator, siteId, envContext);
+            SearchWorker.indexTree(dispatcher, delegator, siteId);
         } catch (Exception e) {
             Debug.logError(e, module);
             return ServiceUtil.returnError(UtilProperties.getMessage(resource,
                     "ContentIndexingTreeError", UtilMisc.toMap("errorString", e.toString()), locale));
         }
-        Date end = new Date();
-        if (Debug.infoOn()) Debug.logInfo("in indexTree, results:" + results, module);
-        if (Debug.infoOn()) Debug.logInfo("Indexing done in: " + (end.getTime()-start.getTime()) + " ms", module);
-        return results;
+        return ServiceUtil.returnSuccess();
     }
 
     public static Map<String, Object> indexProduct(DispatchContext dctx, Map<String, ? extends Object> context) {

Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java Tue Oct 15 10:13:11 2013
@@ -18,16 +18,12 @@
  *******************************************************************************/
 package org.ofbiz.content.search;
 
-import java.io.File;
-import java.io.IOException;
 import java.lang.String;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 import org.ofbiz.base.util.Debug;
 import org.ofbiz.base.util.UtilDateTime;
-import org.ofbiz.base.util.UtilGenerics;
 import org.ofbiz.base.util.UtilMisc;
 import org.ofbiz.base.util.UtilProperties;
 import org.ofbiz.base.util.UtilValidate;
@@ -37,17 +33,6 @@ import org.ofbiz.entity.GenericEntityExc
 import org.ofbiz.entity.GenericValue;
 import org.ofbiz.service.LocalDispatcher;
 
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Version;
 
 /**
@@ -59,9 +44,8 @@ public class SearchWorker {
 
     public static final Version LUCENE_VERSION = Version.LUCENE_45;
 
-    public static Map<String, Object> indexTree(LocalDispatcher dispatcher, Delegator delegator, String siteId, Map<String, Object> context) throws Exception {
+    public static void indexTree(LocalDispatcher dispatcher, Delegator delegator, String siteId) throws Exception {
         GenericValue content = delegator.makeValue("Content", UtilMisc.toMap("contentId", siteId));
-        if (Debug.infoOn()) Debug.logInfo("in indexTree, siteId:" + siteId + " content:" + content, module);
         List<GenericValue> siteList = ContentWorker.getAssociatedContent(content, "To", UtilMisc.toList("SUBSITE", "PUBLISH_LINK", "SUB_CONTENT"), null, UtilDateTime.nowTimestamp().toString(), null);
 
         if (siteList != null) {
@@ -74,19 +58,11 @@ public class SearchWorker {
                     for (GenericValue subContent : subContentList) {
                         contentIdList.add(subContent.getString("contentId"));
                     }
-                    indexContentList(dispatcher, delegator, context, contentIdList);
-                    indexTree(dispatcher, delegator, siteContentId, context);
-                } else {
-                    List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-                    badIndexList.add(siteContentId + " had no sub-entities.");
+                    indexContentList(dispatcher, delegator, contentIdList);
+                    indexTree(dispatcher, delegator, siteContentId);
                 }
             }
-        } else {
-            List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
-            badIndexList.add(siteId + " had no sub-entities.");
         }
-
-        return UtilMisc.toMap("badIndexList", context.get("badIndexList"), "goodIndexCount", context.get("goodIndexCount"));
     }
 
     public static String getIndexPath(String path) {
@@ -94,40 +70,13 @@ public class SearchWorker {
         return (UtilValidate.isNotEmpty(path)? basePath + "/" + path: basePath);
     }
 
-    private static IndexWriter getDefaultIndexWriter(Directory directory) {
-        IndexWriter writer = null;
-        long savedWriteLockTimeout = IndexWriterConfig.getDefaultWriteLockTimeout();
-        Analyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
-        IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
-        IndexWriterConfig.setDefaultWriteLockTimeout(2000);
-        try {
-            writer  = new IndexWriter(directory, conf);
-        } catch (CorruptIndexException e) {
-            Debug.logError("Corrupted lucene index: "  + e.getMessage(), module);
-        } catch (LockObtainFailedException e) {
-            Debug.logError("Could not obtain Lock on lucene index "  + e.getMessage(), module);
-        } catch (IOException e) {
-            Debug.logError(e.getMessage(), module);
-        } finally {
-            IndexWriterConfig.setDefaultWriteLockTimeout(savedWriteLockTimeout);
-        }
-        return writer;
-    }
-
-    public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, Map<String, Object> context,List<String> idList) throws Exception {
-        Directory directory = FSDirectory.open(new File(getIndexPath("content")));
-        if (Debug.infoOn()) Debug.logInfo("in indexContentList, indexAllPath: " + directory.toString(), module);
-        // Delete existing documents
-        IndexWriter writer = getDefaultIndexWriter(directory);
+    public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, List<String> idList) throws Exception {
+        DocumentIndexer indexer = DocumentIndexer.getInstance(delegator, "content");
         List<GenericValue> contentList = new ArrayList<GenericValue>();
         for (String id : idList) {
-            if (Debug.infoOn()) Debug.logInfo("in indexContentList, id:" + id, module);
             try {
                 GenericValue content = delegator.findOne("Content", UtilMisc .toMap("contentId", id), true);
                 if (content != null) {
-                    if (writer != null) {
-                        deleteContentDocuments(content, writer);
-                    }
                     contentList.add(content);
                 }
             } catch (GenericEntityException e) {
@@ -135,50 +84,8 @@ public class SearchWorker {
                 return;
             }
         }
-        for (GenericValue gv : contentList) {
-            indexContent(dispatcher, context, gv, writer);
+        for (GenericValue content : contentList) {
+            indexer.queue(new ContentDocument(content, dispatcher));
         }
-        try {
-            writer.forceMerge(1);
-        } catch (NullPointerException e) {
-            Debug.logError(e, module);
-        }
-        writer.close();
     }
-
-    private static void deleteContentDocuments(GenericValue content, IndexWriter writer) throws Exception {
-        String contentId = content.getString("contentId");
-        Term term = new Term("contentId", contentId);
-        deleteDocumentsByTerm(term, writer);
-        String dataResourceId = content.getString("dataResourceId");
-        if (dataResourceId != null) {
-            term = new Term("dataResourceId", dataResourceId);
-            deleteDocumentsByTerm(term, writer);
-        }
-    }
-
-    private static void deleteDocumentsByTerm(Term term, IndexWriter writer) throws Exception {
-        DirectoryReader reader = DirectoryReader.open(writer, false);
-        int qtyBefore = reader.docFreq(term);
-
-        //deletes documents, all the rest is for logging
-        writer.deleteDocuments(term);
-
-        int qtyAfter = reader.docFreq(term);
-        reader.close();
-
-        if (Debug.infoOn()) Debug.logInfo("For term " + term.toString() + ", documents deleted: " + qtyBefore + ", remaining: " + qtyAfter, module);
-    }
-
-    private static void indexContent(LocalDispatcher dispatcher, Map<String, Object> context, GenericValue content, IndexWriter writer) throws Exception {
-        Document doc = ContentDocument.Document(content, context, dispatcher);
-
-        if (doc != null) {
-            writer.addDocument(doc);
-            Integer goodIndexCount = (Integer)context.get("goodIndexCount");
-            Integer newIndexCount = goodIndexCount + 1;
-            context.put("goodIndexCount", newIndexCount);
-        }
-    }
-
 }