Re: svn commit: r1531918 - in /ofbiz/trunk/specialpurpose/lucene: entitydef/ entitydef/eecas_product.xml ofbiz-component.xml servicedef/services.xml src/org/ofbiz/content/search/ProductIndexer.java src/org/ofbiz/content/search/SearchServices.java

[Jacques Le Roux]

Hi Jacopo, Scott,

Could you please have a look at this commit in relation with OFBIZ-5385 and OFBIZ-5386 (see last comments in OFBIZ-5389)
I noticed some related changes in http://svn.apache.org/viewvc?view=revision&revision=r1532241 also

Thanks

Jacques

[hidden email] wrote:

> Author: jacopoc
> Date: Mon Oct 14 14:42:53 2013
> New Revision: 1531918
>
> URL: http://svn.apache.org/r1531918
> Log:
> Implemented a series of ecas to submit for indexing products when product related information is added/updated/deleted.
> Thanks to Scott Gray for the initial implementation of these ecas.
>
> Added:
>    ofbiz/trunk/specialpurpose/lucene/entitydef/
>    ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml   (with props)
> Modified:
>    ofbiz/trunk/specialpurpose/lucene/ofbiz-component.xml
>    ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
>    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java
>    ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
>
> Added: ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml
> URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml?rev=1531918&view=auto
> ==============================================================================
> --- ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml (added)
> +++ ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml Mon Oct 14 14:42:53 2013
> @@ -0,0 +1,72 @@
> +<?xml version="1.0" encoding="UTF-8"?>
> +<!--
> +Licensed to the Apache Software Foundation (ASF) under one
> +or more contributor license agreements.  See the NOTICE file
> +distributed with this work for additional information
> +regarding copyright ownership.  The ASF licenses this file
> +to you under the Apache License, Version 2.0 (the
> +"License"); you may not use this file except in compliance
> +with the License.  You may obtain a copy of the License at
> +
> +http://www.apache.org/licenses/LICENSE-2.0
> +
> +Unless required by applicable law or agreed to in writing,
> +software distributed under the License is distributed on an
> +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> +KIND, either express or implied.  See the License for the
> +specific language governing permissions and limitations
> +under the License.
> +-->
> +
> +<entity-eca xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
> +        xsi:noNamespaceSchemaLocation="http://ofbiz.apache.org/dtds/entity-eca.xsd">
> +
> +    <!-- Product Keyword Indexing ECAs -->
> +    <eca entity="Product" operation="create-store-remove" event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductFeatureAppl" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductAttribute" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="GoodIndentification" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductContent" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductCategoryMember" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductPrice" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +    <eca entity="SupplierProduct" operation="create-store-remove"  event="return">
> +        <action service="indexProduct" mode="sync"/>
> +    </eca>
> +
> +    <eca entity="ProductAssoc" operation="create-store-remove"  event="return">
> +        <action service="indexProductsFromProductAssoc" mode="sync"/>
> +    </eca>
> +    <eca entity="DataResource" operation="store"  event="return">
> +        <action service="indexProductsFromDataResource" mode="sync"/>
> +    </eca>
> +    <eca entity="Content" operation="store"  event="return">
> +        <action service="indexProductsFromContent" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductCategoryRollup" operation="create-store-remove"  event="return">
> +        <action service="indexProductsFromCategory" mode="sync"/>
> +    </eca>
> +    <eca entity="ProdCatalogCategory" operation="create-store-remove"  event="return">
> +        <action service="indexProductsFromCategory" mode="sync"/>
> +    </eca>
> +    <!-- Don't need to index on create or remove since the feature won't be attached to any products at those points -->
> +    <eca entity="ProductFeature" operation="store"  event="return">
> +        <action service="indexProductsFromFeature" mode="sync"/>
> +    </eca>
> +    <eca entity="ProductFeatureGroupAppl" operation="create-store-remove"  event="return">
> +        <action service="indexProductsFromFeature" mode="sync"/>
> +    </eca>
> +</entity-eca>
>
> Propchange: ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml
> ------------------------------------------------------------------------------
>    svn:eol-style = native
>
> Propchange: ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml
> ------------------------------------------------------------------------------
>    svn:keywords = Date Rev Author URL Id
>
> Propchange: ofbiz/trunk/specialpurpose/lucene/entitydef/eecas_product.xml
> ------------------------------------------------------------------------------
>    svn:mime-type = text/xml
>
> Modified: ofbiz/trunk/specialpurpose/lucene/ofbiz-component.xml
> URL:
> http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/ofbiz-component.xml?rev=1531918&r1=1531917&r2=1531918&view=diff
> ============================================================================== ---
> ofbiz/trunk/specialpurpose/lucene/ofbiz-component.xml (original) +++ ofbiz/trunk/specialpurpose/lucene/ofbiz-component.xml Mon
> Oct 14 14:42:53 2013 @@ -30,6 +30,9 @@ under the License.
>     <classpath type="jar" location="build/lib/*"/>
>     <classpath type="jar" location="lib/*"/>
>
> +    <!-- entity resources: model(s), eca(s), group, and data definitions -->
> +    <entity-resource type="eca" reader-name="main" loader="main" location="entitydef/eecas_product.xml"/>
> +
>     <!-- service resources: model(s), eca(s) and group definitions -->
>     <service-resource type="model" loader="main" location="servicedef/services.xml"/>
>
>
> Modified: ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
> URL:
> http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml?rev=1531918&r1=1531917&r2=1531918&view=diff
> ============================================================================== ---
> ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml (original) +++
> ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml Mon Oct 14 14:42:53 2013 @@ -30,5 +30,28 @@ under the License.
>         <attribute mode="OUT" name="badIndexList" optional="true" type="List"/>
>         <attribute mode="OUT" name="goodIndexCount" optional="true" type="Integer"/>
>     </service>
> -
> +
> +    <service name="indexProduct" engine="java" location="org.ofbiz.content.search.SearchServices" invoke="indexProduct">
> +        <description>Submit for indexing the specified productId</description>
> +        <attribute name="productId" mode="IN" type="String" optional="false"/>
> +    </service>
> +
> +    <!-- Services to locate and submit for indexing all products affected by an associated data change -->
> +    <service name="indexProductsFromFeature" engine="java" location="org.ofbiz.content.search.SearchServices"
> invoke="indexProductsFromFeature"> +        <attribute name="productFeatureId" mode="IN" type="String" optional="false"/>
> +    </service>
> +    <service name="indexProductsFromProductAssoc" engine="java" location="org.ofbiz.content.search.SearchServices"
> invoke="indexProductsFromProductAssoc"> +        <attribute name="productId" mode="IN" type="String"/>
> +        <attribute name="productIdTo" mode="IN" type="String"/>
> +    </service>
> +    <service name="indexProductsFromDataResource" engine="java" location="org.ofbiz.content.search.SearchServices"
> invoke="indexProductsFromDataResource"> +        <attribute name="dataResourceId" mode="IN" type="String"/>
> +    </service>
> +    <service name="indexProductsFromContent" engine="java" location="org.ofbiz.content.search.SearchServices"
> invoke="indexProductsFromContent"> +        <attribute name="contentId" mode="IN" type="String"/>
> +    </service>
> +    <service name="indexProductsFromCategory" engine="java" location="org.ofbiz.content.search.SearchServices"
> invoke="indexProductsFromCategory"> +        <attribute name="productCategoryId" mode="IN" type="String"/>
> +    </service>
> +
> </services>
> \ No newline at end of file
>
> Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java
> URL:
> http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java?rev=1531918&r1=1531917&r2=1531918&view=diff
> ============================================================================== ---
> ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java (original) +++
> ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java Mon Oct 14 14:42:53 2013 @@ -28,8 +28,6 @@
> import java.util.Set;
> import java.util.TreeSet;
> import java.util.concurrent.LinkedBlockingQueue;
>
> -import org.apache.lucene.document.IntField;
> -import org.apache.lucene.document.LongField;
> import org.ofbiz.base.util.Debug;
> import org.ofbiz.base.util.GeneralException;
> import org.ofbiz.base.util.UtilDateTime;
> @@ -45,11 +43,11 @@ import org.ofbiz.entity.condition.Entity
> import org.ofbiz.entity.util.EntityUtil;
>
> import org.apache.lucene.analysis.standard.StandardAnalyzer;
> -import org.apache.lucene.analysis.Analyzer;
> import org.apache.lucene.document.Document;
> import org.apache.lucene.document.DoubleField;
> import org.apache.lucene.document.Field;
> import org.apache.lucene.document.Field.Store;
> +import org.apache.lucene.document.LongField;
> import org.apache.lucene.document.StringField;
> import org.apache.lucene.document.TextField;
> import org.apache.lucene.index.CorruptIndexException;
> @@ -68,15 +66,12 @@ public class ProductIndexer extends Thre
>     private LinkedBlockingQueue<String> productIndexQueue = new LinkedBlockingQueue<String>();
>     private Delegator delegator;
>     private Directory indexDirectory;
> -    private IndexWriterConfig indexWriterConfiguration;
>     private static final String NULL_STRING = "NULL";
>     // TODO: Move to property file
>     private static final int UNCOMMITTED_DOC_LIMIT = 100;
>
>     private ProductIndexer(Delegator delegator) {
>         this.delegator = delegator;
> -        Analyzer analyzer = new StandardAnalyzer(SearchWorker.LUCENE_VERSION);
> -        this.indexWriterConfiguration = new IndexWriterConfig(SearchWorker.LUCENE_VERSION, analyzer);
>         try {
>             this.indexDirectory = FSDirectory.open(new File(SearchWorker.getIndexPath("products")));
>         } catch (CorruptIndexException e) {
> @@ -124,7 +119,7 @@ public class ProductIndexer extends Thre
>             Term documentIdentifier = new Term("productId", productId);
>             if (indexWriter == null) {
>                 try {
> -                    indexWriter  = new IndexWriter(this.indexDirectory, this.indexWriterConfiguration);
> +                    indexWriter  = new IndexWriter(this.indexDirectory, new IndexWriterConfig(SearchWorker.LUCENE_VERSION, new
>                 StandardAnalyzer(SearchWorker.LUCENE_VERSION))); } catch (CorruptIndexException e) {
>                     Debug.logError("Corrupted lucene index: "  + e.getMessage(), module);
>                     break;
>
> Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
> URL:
> http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java?rev=1531918&r1=1531917&r2=1531918&view=diff
> ============================================================================== ---
> ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java (original) +++
> ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java Mon Oct 14 14:42:53 2013 @@ -26,12 +26,17 @@
> import java.util.HashMap;
> import java.util.List;
> import java.util.Locale;
> import java.util.Map;
> +import java.util.Set;
>
> import org.ofbiz.base.util.Debug;
> import org.ofbiz.base.util.UtilMisc;
> import org.ofbiz.base.util.UtilProperties;
> import org.ofbiz.entity.Delegator;
> +import org.ofbiz.entity.GenericEntityException;
> +import org.ofbiz.entity.GenericValue;
> +import org.ofbiz.entity.util.EntityUtil;
> import org.ofbiz.service.DispatchContext;
> +import org.ofbiz.service.GenericServiceException;
> import org.ofbiz.service.ServiceUtil;
> import org.ofbiz.service.LocalDispatcher;
>
> @@ -69,4 +74,117 @@ public class SearchServices {
>         if (Debug.infoOn()) Debug.logInfo("Indexing done in: " + (end.getTime()-start.getTime()) + " ms", module);
>         return results;
>     }
> +
> +    public static Map<String, Object> indexProduct(DispatchContext dctx, Map<String, ? extends Object> context) {
> +        Delegator delegator = dctx.getDelegator();
> +        String productId = (String) context.get("productId");
> +        ProductIndexer indexer = ProductIndexer.getInstance(delegator);
> +        indexer.queue(productId);
> +        return ServiceUtil.returnSuccess();
> +    }
> +
> +    public static Map<String, Object> indexProductsFromFeature(DispatchContext dctx, Map<String, ? extends Object> context) {
> +        Delegator delegator = dctx.getDelegator();
> +        LocalDispatcher dispatcher = dctx.getDispatcher();
> +        try {
> +            List<GenericValue> productFeatureAppls = delegator.findByAnd("ProductFeatureAppl",
> UtilMisc.toMap("productFeatureId", context.get("productFeatureId")), null, false); +            // Only re-index the active
> appls, future dated ones will get picked up on that product's re-index date +            productFeatureAppls =
> EntityUtil.filterByDate(productFeatureAppls); +
> +            for (GenericValue productFeatureAppl : productFeatureAppls) {
> +                try {
> +                    dispatcher.runSync("indexProduct", UtilMisc.toMap("productId", productFeatureAppl.get("productId")));
> +                } catch (GenericServiceException e) {
> +                    Debug.logError(e, module);
> +                }
> +            }
> +
> +        } catch (GenericEntityException e) {
> +            Debug.logError(e, module);
> +        }
> +        return ServiceUtil.returnSuccess();
> +    }
> +
> +    public static Map<String, Object> indexProductsFromProductAssoc(DispatchContext dctx, Map<String, ? extends Object> context)
> { +        LocalDispatcher dispatcher = dctx.getDispatcher();
> +        try {
> +            dispatcher.runSync("indexProduct", UtilMisc.toMap("productId", context.get("productId")));
> +            dispatcher.runSync("indexProduct", UtilMisc.toMap("productId", context.get("productIdTo")));
> +        } catch (GenericServiceException e) {
> +            Debug.logError(e, module);
> +        }
> +        return ServiceUtil.returnSuccess();
> +    }
> +
> +    public static Map<String, Object> indexProductsFromDataResource(DispatchContext dctx, Map<String, ? extends Object> context)
> { +        Delegator delegator = dctx.getDelegator();
> +        LocalDispatcher dispatcher = dctx.getDispatcher();
> +        try {
> +            List<GenericValue> contents = delegator.findByAnd("Content", UtilMisc.toMap("dataResourceId",
> context.get("dataResourceId")), null, false); +            for (GenericValue content : contents) {
> +                dispatcher.runSync("indexProductsFromContent",
> +                        UtilMisc.toMap(
> +                                "userLogin", context.get("userLogin"),
> +                                "contentId", content.get("contentId")));
> +            }
> +        } catch (GenericEntityException e) {
> +            Debug.logError(e, module);
> +        } catch (GenericServiceException e) {
> +            Debug.logError(e, module);
> +        }
> +        return ServiceUtil.returnSuccess();
> +    }
> +    public static Map<String, Object> indexProductsFromContent(DispatchContext dctx, Map<String, ? extends Object> context) {
> +        Delegator delegator = dctx.getDelegator();
> +        LocalDispatcher dispatcher = dctx.getDispatcher();
> +        try {
> +            List<GenericValue> productContents = delegator.findByAnd("ProductContent", UtilMisc.toMap("contentId",
> context.get("contentId")), null, false); +            for (GenericValue productContent : productContents) {
> +                try {
> +                    dispatcher.runSync("indexProduct", UtilMisc.toMap("productId", productContent.get("productId")));
> +                } catch (GenericServiceException e) {
> +                    Debug.logError(e, module);
> +                }
> +            }
> +        } catch (GenericEntityException e) {
> +            Debug.logError(e, module);
> +        }
> +        return ServiceUtil.returnSuccess();
> +    }
> +    public static Map<String, Object> indexProductsFromCategory(DispatchContext dctx, Map<String, Object> context) {
> +        Delegator delegator = dctx.getDelegator();
> +        LocalDispatcher dispatcher = dctx.getDispatcher();
> +        try {
> +            String productCategoryId = (String) context.get("productCategoryId");
> +            indexProductCategoryMembers(productCategoryId, delegator, dispatcher);
> +            indexProductCategoryRollup(productCategoryId, delegator, dispatcher, UtilMisc.<String>toSet(productCategoryId));
> +        } catch (GenericEntityException e) {
> +            Debug.logError(e, module);
> +        }
> +        return ServiceUtil.returnSuccess();
> +    }
> +    private static void indexProductCategoryRollup(String parentProductCategoryId, Delegator delegator, LocalDispatcher
> dispatcher, Set<String> excludeProductCategoryIds) throws GenericEntityException { +        List<GenericValue>
> productCategoryRollups = delegator.findByAnd("ProductCategoryRollup", UtilMisc.toMap("parentProductCategoryId",
> parentProductCategoryId), null, false); +        for (GenericValue productCategoryRollup : productCategoryRollups) { +          
> String productCategoryId = productCategoryRollup.getString("productCategoryId"); +            // Avoid infinite recursion
> +            if (!excludeProductCategoryIds.add(productCategoryId)) {
> +                continue;
> +            }
> +            indexProductCategoryMembers(productCategoryId, delegator, dispatcher);
> +            indexProductCategoryRollup(productCategoryId, delegator, dispatcher, excludeProductCategoryIds);
> +        }
> +    }
> +
> +    private static void indexProductCategoryMembers(String productCategoryId, Delegator delegator, LocalDispatcher dispatcher)
> throws GenericEntityException { +        List<GenericValue> productCategoryMembers = delegator.findByAnd("ProductCategoryMember",
> UtilMisc.toMap("productCategoryId", productCategoryId), null, false); +        for (GenericValue productCategoryMember :
> productCategoryMembers) { +            try {
> +                dispatcher.runSync("indexProduct", UtilMisc.toMap("productId", productCategoryMember.get("productId")));
> +            } catch (GenericServiceException e) {
> +                Debug.logError(e, module);
> +            }
> +        }
> +    }
> +
> }