OFBiz › OFBiz - Commits

svn commit: r1812790 - in /ofbiz/ofbiz-plugins/trunk: lucene/ lucene/src/main/java/org/apache/ofbiz/content/search/ solr/ solr/src/main/java/org/apache/ofbiz/solr/ solr/src/main/java/org/apache/ofbiz/solr/test/ solr/testdef/ solr/testdef/data/

_‹ Previous Topic Next Topic _›

Classic

List

Threaded

1 message

mbrohl

svn commit: r1812790 - in /ofbiz/ofbiz-plugins/trunk: lucene/ lucene/src/main/java/org/apache/ofbiz/content/search/ solr/ solr/src/main/java/org/apache/ofbiz/solr/ solr/src/main/java/org/apache/ofbiz/solr/test/ solr/testdef/ solr/testdef/data/

Author: mbrohl
Date: Fri Oct 20 20:48:01 2017
New Revision: 1812790

URL: http://svn.apache.org/viewvc?rev=1812790&view=rev
Log:
Improved: Update Solr/Lucene to 7.1.0 [CVE-2017-9803] [CVE-2017-12629].
(OFBIZ-9835)

There were two security vulnerabilities and it was recommended to
upgrade to 7.1.0 asap.

The update removed the deprecated setBoost functionality during index
time. I have removed it completely to be able to update to 7.1.0.
A new indexing and query strategy has to be implemented here. Might be
follow-up work in the future.

Added:
ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/
ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java (with props)
ofbiz/ofbiz-plugins/trunk/solr/testdef/
ofbiz/ofbiz-plugins/trunk/solr/testdef/data/
ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml (with props)
ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml (with props)
Modified:
ofbiz/ofbiz-plugins/trunk/lucene/build.gradle
ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/ProductDocument.java
ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/SearchWorker.java
ofbiz/ofbiz-plugins/trunk/solr/build.gradle
ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/SolrUtil.java

Modified: ofbiz/ofbiz-plugins/trunk/lucene/build.gradle
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/lucene/build.gradle?rev=1812790&r1=1812789&r2=1812790&view=diff
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/lucene/build.gradle (original)
+++ ofbiz/ofbiz-plugins/trunk/lucene/build.gradle Fri Oct 20 20:48:01 2017
@@ -17,7 +17,7 @@
* under the License.
*/
dependencies {
- pluginLibsCompile 'org.apache.lucene:lucene-core:6.6.0'
- pluginLibsCompile 'org.apache.lucene:lucene-queryparser:6.6.0'
- pluginLibsCompile 'org.apache.lucene:lucene-analyzers-common:6.6.0'
+ pluginLibsCompile 'org.apache.lucene:lucene-core:7.1.0'
+ pluginLibsCompile 'org.apache.lucene:lucene-queryparser:7.1.0'
+ pluginLibsCompile 'org.apache.lucene:lucene-analyzers-common:7.1.0'
}
\ No newline at end of file

Modified: ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/ProductDocument.java
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/ProductDocument.java?rev=1812790&r1=1812789&r2=1812790&view=diff
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/ProductDocument.java (original)
+++ ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/ProductDocument.java Fri Oct 20 20:48:01 2017
@@ -81,11 +81,11 @@ public class ProductDocument implements

// Product Fields
doc.add(new StringField("productId", productId, Field.Store.YES));
- this.addTextFieldByWeight(doc, "productName", product.getString("productName"), "index.weight.Product.productName", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "internalName", product.getString("internalName"), "index.weight.Product.internalName", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "brandName", product.getString("brandName"), "index.weight.Product.brandName", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "description", product.getString("description"), "index.weight.Product.description", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "longDescription", product.getString("longDescription"), "index.weight.Product.longDescription", 0, false, "fullText", delegator);
+ this.addTextField(doc, "productName", product.getString("productName"), false, "fullText", delegator);
+ this.addTextField(doc, "internalName", product.getString("internalName"), false, "fullText", delegator);
+ this.addTextField(doc, "brandName", product.getString("brandName"), false, "fullText", delegator);
+ this.addTextField(doc, "description", product.getString("description"), false, "fullText", delegator);
+ this.addTextField(doc, "longDescription", product.getString("longDescription"), false, "fullText", delegator);
doc.add(new LongPoint("introductionDate", quantizeTimestampToDays(product.getTimestamp("introductionDate"))));
nextReIndex = this.checkSetNextReIndex(product.getTimestamp("introductionDate"), nextReIndex);
doc.add(new LongPoint("salesDiscontinuationDate", quantizeTimestampToDays(product.getTimestamp("salesDiscontinuationDate"))));
@@ -113,9 +113,9 @@ public class ProductDocument implements
doc.add(new StringField("productFeatureId", productFeatureAndAppl.getString("productFeatureId"), Field.Store.NO));
doc.add(new StringField("productFeatureCategoryId", productFeatureAndAppl.getString("productFeatureCategoryId"), Field.Store.NO));
doc.add(new StringField("productFeatureTypeId", productFeatureAndAppl.getString("productFeatureTypeId"), Field.Store.NO));
- this.addTextFieldByWeight(doc, "featureDescription", productFeatureAndAppl.getString("description"), "index.weight.ProductFeatureAndAppl.description", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "featureAbbreviation", productFeatureAndAppl.getString("abbrev"), "index.weight.ProductFeatureAndAppl.abbrev", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "featureCode", productFeatureAndAppl.getString("idCode"), "index.weight.ProductFeatureAndAppl.idCode", 0, false, "fullText", delegator);
+ this.addTextField(doc, "featureDescription", productFeatureAndAppl.getString("description"), false, "fullText", delegator);
+ this.addTextField(doc, "featureAbbreviation", productFeatureAndAppl.getString("abbrev"), false, "fullText", delegator);
+ this.addTextField(doc, "featureCode", productFeatureAndAppl.getString("idCode"), false, "fullText", delegator);
// Get the ProductFeatureGroupIds
List<GenericValue> productFeatureGroupAppls = EntityQuery.use(delegator).from("ProductFeatureGroupAppl").where("productFeatureId", productFeatureAndAppl.get("productFeatureId")).queryList();
productFeatureGroupAppls = this.filterByThruDate(productFeatureGroupAppls);
@@ -140,8 +140,8 @@ public class ProductDocument implements

List<GenericValue> productAttributes = EntityQuery.use(delegator).from("ProductAttribute").where("productId", productId).queryList();
for (GenericValue productAttribute: productAttributes) {
- this.addTextFieldByWeight(doc, "attributeName", productAttribute.getString("attrName"), "index.weight.ProductAttribute.attrName", 0, false, "fullText", delegator);
- this.addTextFieldByWeight(doc, "attributeValue", productAttribute.getString("attrValue"), "index.weight.ProductAttribute.attrValue", 0, false, "fullText", delegator);
+ this.addTextField(doc, "attributeName", productAttribute.getString("attrName"), false, "fullText", delegator);
+ this.addTextField(doc, "attributeValue", productAttribute.getString("attrValue"), false, "fullText", delegator);
}
}

@@ -153,7 +153,7 @@ public class ProductDocument implements
String idValue = goodIdentification.getString("idValue");
doc.add(new StringField("goodIdentificationTypeId", goodIdentificationTypeId, Field.Store.NO));
doc.add(new StringField(goodIdentificationTypeId + "_GoodIdentification", idValue, Field.Store.NO));
- this.addTextFieldByWeight(doc, "identificationValue", idValue, "index.weight.GoodIdentification.idValue", 0, false, "fullText", delegator);
+ this.addTextField(doc, "identificationValue", idValue, false, "fullText", delegator);
}
}

@@ -172,7 +172,7 @@ public class ProductDocument implements
} else if (thruDate != null) {
nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
}
- this.addTextFieldByWeight(doc, "variantProductId", variantProductAssoc.getString("productIdTo"), "index.weight.Variant.Product.productId", 0, false, "fullText", delegator);
+ this.addTextField(doc, "variantProductId", variantProductAssoc.getString("productIdTo"), false, "fullText", delegator);
}
}
}
@@ -203,7 +203,7 @@ public class ProductDocument implements
try {
Map<String, Object> drContext = UtilMisc.<String, Object>toMap("product", product);
String contentText = DataResourceWorker.renderDataResourceAsText(null, delegator, productContentAndInfo.getString("dataResourceId"), drContext, null, null, false);
- this.addTextFieldByWeight(doc, "content", contentText, null, weight, false, "fullText", delegator);
+ this.addTextField(doc, "content", contentText, false, "fullText", delegator);
} catch (IOException e1) {
Debug.logError(e1, "Error getting content text to index", module);
} catch (GeneralException e1) {
@@ -279,26 +279,10 @@ public class ProductDocument implements
}

// An attempt to boost/weight values in a similar manner to what OFBiz product search does.
- private void addTextFieldByWeight(Document doc, String fieldName, String value, String property, int defaultWeight, boolean store, String fullTextFieldName, Delegator delegator) {
+ private void addTextField(Document doc, String fieldName, String value, boolean store, String fullTextFieldName, Delegator delegator) {
if (fieldName == null) return;

- float weight = 0;
- if (property != null) {
- try {
- weight = EntityUtilProperties.getPropertyAsFloat("prodsearch", property, 0).floatValue();
- } catch (Exception e) {
- Debug.logWarning("Could not parse weight number: " + e.toString(), module);
- }
- } else if (defaultWeight > 0) {
- weight = defaultWeight;
- }
- if (weight == 0 && !store) {
- return;
- }
Field field = new TextField(fieldName, checkValue(value), (store? Field.Store.YES: Field.Store.NO));
- if (weight > 0 && weight != 1) {
- field.setBoost(weight);
- }
doc.add(field);
if (fullTextFieldName != null) {
doc.add(new TextField(fullTextFieldName, checkValue(value), Field.Store.NO));

Modified: ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/SearchWorker.java
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/SearchWorker.java?rev=1812790&r1=1812789&r2=1812790&view=diff
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/SearchWorker.java (original)
+++ ofbiz/ofbiz-plugins/trunk/lucene/src/main/java/org/apache/ofbiz/content/search/SearchWorker.java Fri Oct 20 20:48:01 2017
@@ -41,7 +41,7 @@ public final class SearchWorker {

public static final String module = SearchWorker.class.getName();

- private static final Version LUCENE_VERSION = Version.LUCENE_6_6_0;
+ private static final Version LUCENE_VERSION = Version.LUCENE_7_1_0;

private SearchWorker() {}

Modified: ofbiz/ofbiz-plugins/trunk/solr/build.gradle
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/solr/build.gradle?rev=1812790&r1=1812789&r2=1812790&view=diff
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/solr/build.gradle (original)
+++ ofbiz/ofbiz-plugins/trunk/solr/build.gradle Fri Oct 20 20:48:01 2017
@@ -17,7 +17,7 @@
* under the License.
*/
dependencies {
- pluginLibsCompile 'org.apache.solr:solr-core:6.6.0'
+ pluginLibsCompile 'org.apache.solr:solr-core:7.1.0'
pluginLibsCompile 'com.google.guava:guava:20.0'
}

Modified: ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/SolrUtil.java
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/SolrUtil.java?rev=1812790&r1=1812789&r2=1812790&view=diff
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/SolrUtil.java (original)
+++ ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/SolrUtil.java Fri Oct 20 20:48:01 2017
@@ -330,7 +330,7 @@ public final class SolrUtil {
httpLogin.setConfig(requestConfig);
CloseableHttpResponse loginResponse = httpClient.execute(httpLogin, httpContext);
loginResponse.close();
- return new HttpSolrClient(solrUrl + "/" + solrIndexName, httpClient);
+ return new HttpSolrClient.Builder(solrUrl + "/" + solrIndexName).withHttpClient(httpClient).build();
}

}

Added: ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java?rev=1812790&view=auto
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java (added)
+++ ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java Fri Oct 20 20:48:01 2017
@@ -0,0 +1,141 @@
+
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.ofbiz.solr.test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.jasper.tagplugins.jstl.core.Remove;
+import org.apache.ofbiz.base.util.UtilMisc;
+import org.apache.ofbiz.entity.Delegator;
+import org.apache.ofbiz.entity.GenericValue;
+import org.apache.ofbiz.entity.util.EntityQuery;
+import org.apache.ofbiz.service.ServiceUtil;
+import org.apache.ofbiz.service.testtools.OFBizTestCase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.client.solrj.response.QueryResponse;
+
+public class SolrTests extends OFBizTestCase {
+
+ protected GenericValue userLogin = null;
+ private Map<String, Object> context;
+ private Map<String, Object> emptyContext = new HashMap<String, Object>();
+ private Map<String, Object> response;
+ private String validTestProductId = "GZ-1006";
+ private String validTestProductId_2 = "GZ-1005";
+ private String invalidTestProductId = validTestProductId + validTestProductId;
+
+ public SolrTests(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ userLogin = EntityQuery.use(delegator).from("UserLogin").where("userLoginId", "system").queryOne();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ }
+
+ public void testAddProductToIndex() throws Exception {
+
+ GenericValue product = delegator.findOne("Product", UtilMisc.toMap("productId", validTestProductId), false);
+
+ Map<String, Object> ctx = new HashMap<String, Object>();
+ ctx.put("instance", product);
+
+ Map<String, Object> resp = dispatcher.runSync("addToSolr", ctx);
+ assertTrue("Could not init search index", ServiceUtil.isSuccess(resp));
+
+ Map<String, Object> sctx = new HashMap<String, Object>();
+ sctx.put("productCategoryId", "102");
+
+ Map<String, Object> sresp = dispatcher.runSync("solrProductsSearch", sctx);
+ assertTrue("Could not query search index", ServiceUtil.isSuccess(sresp));
+
+
+ }
+
+ public void testAddToSolrIndex() throws Exception{
+ context = new HashMap<>();
+ context.put("productId", validTestProductId);
+ response = dispatcher.runSync("addToSolrIndex", context);
+ assertTrue("Could not add Product to Index", ServiceUtil.isSuccess(
+ response));
+ }
+
+ public void testAddToSolrIndex_invalidProduct() throws Exception {
+ context = new HashMap<>();
+ context.put("productId", invalidTestProductId);
+ response = dispatcher.runSync("addToSolrIndex", context);
+ assertTrue("Could not test the addition of an invalid product to the Solr index", ServiceUtil.isSuccess(
+ response));
+ }
+
+ public void testAddListToSolrIndex() throws Exception {
+ List<Map<String, Object>> products = new ArrayList<>();
+ Map<String, Object> product_1 = new HashMap<>();
+ Map<String, Object> product_2 = new HashMap<>();
+ GenericValue validTestProduct = delegator.findOne("Product", UtilMisc.toMap("productId", validTestProductId), false);
+ GenericValue validTestProduct_2 = delegator.findOne("Product", UtilMisc.toMap("productId", validTestProductId_2), false);
+
+ product_1.put("productId", validTestProduct);
+ product_2.put("productId", validTestProduct_2);
+
+ products.add(product_1);
+ products.add(product_2);
+ context = new HashMap<>();
+ context.put("fieldList", products);
+
+ response = dispatcher.runSync("addListToSolrIndex", context);
+ assertTrue("Could not add products to index", ServiceUtil.isSuccess(response));
+
+ }
+
+ public void testAddListToSolrIndex_invalidProducts() throws Exception {
+ List<Map<String, Object>> products = new ArrayList<>();
+ Map<String, Object> product_1 = new HashMap<>();
+ Map<String, Object> product_2 = new HashMap<>();
+ GenericValue testProduct = delegator.findOne("Product", UtilMisc.toMap("productId", validTestProductId), false);
+ GenericValue testProduct_2 = delegator.findOne("Product", UtilMisc.toMap("productId", validTestProductId_2), false);
+
+ testProduct.replace("productId", invalidTestProductId);
+ testProduct.replace("productId", invalidTestProductId);
+
+ product_1.put("productId", testProduct);
+ product_2.put("productId", testProduct_2);
+
+ products.add(product_1);
+ products.add(product_2);
+ context = new HashMap<>();
+ context.put("fieldList", products);
+
+ response = dispatcher.runSync("addListToSolrIndex", context);
+ assertTrue("Could not test adding invalid products to index", ServiceUtil.isSuccess(response));
+
+ }
+}
+

Propchange: ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java
------------------------------------------------------------------------------
svn:eol-style = native

Propchange: ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id

Propchange: ofbiz/ofbiz-plugins/trunk/solr/src/main/java/org/apache/ofbiz/solr/test/SolrTests.java
------------------------------------------------------------------------------
svn:mime-type = text/plain

Added: ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml?rev=1812790&view=auto
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml (added)
+++ ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml Fri Oct 20 20:48:01 2017
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<entity-engine-xml>
+
+
+</entity-engine-xml>
\ No newline at end of file

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml
------------------------------------------------------------------------------
svn:eol-style = native

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/data/SolrTestsData.xml
------------------------------------------------------------------------------
svn:mime-type = text/xml

Added: ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml
URL: http://svn.apache.org/viewvc/ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml?rev=1812790&view=auto
==============================================================================
--- ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml (added)
+++ ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml Fri Oct 20 20:48:01 2017
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<test-suite suite-name="solrtests"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:noNamespaceSchemaLocation="http://ofbiz.apache.org/dtds/test-suite.xsd">
+
+ <test-case case-name="solr-tests-data-load">
+ <entity-xml action="load" entity-xml-url="component://solr/testdef/data/SolrTestsData.xml"/>
+ </test-case>
+
+ <test-case case-name="solr-tests"><junit-test-suite class-name="org.apache.ofbiz.solr.test.SolrTests"/></test-case>
+
+</test-suite>
\ No newline at end of file

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml
------------------------------------------------------------------------------
svn:eol-style = native

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id

Propchange: ofbiz/ofbiz-plugins/trunk/solr/testdef/solrtests.xml
------------------------------------------------------------------------------
svn:mime-type = text/xml