Author: jonesde
Date: Fri Aug 10 04:47:37 2007 New Revision: 564565 URL: http://svn.apache.org/viewvc?view=rev&rev=564565 Log: Implemented a new feature to do a check on data files with no changes to the database instead of loading the data; this includes some refactoring of other classes that do this and a few improvements as well based on some testing Modified: ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntityDataAssert.java ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntitySaxReader.java ofbiz/trunk/framework/webtools/config/WebtoolsUiLabels.properties ofbiz/trunk/framework/webtools/servicedef/services.xml ofbiz/trunk/framework/webtools/src/org/ofbiz/webtools/WebToolsServices.java ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImport.ftl ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportDir.ftl ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportReaders.ftl Modified: ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntityDataAssert.java URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntityDataAssert.java?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntityDataAssert.java (original) +++ ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntityDataAssert.java Fri Aug 10 04:47:37 2007 @@ -57,26 +57,7 @@ Iterator checkValueIter = checkValueList.iterator(); while (checkValueIter.hasNext()) { GenericValue checkValue = (GenericValue) checkValueIter.next(); - - // to check get the PK, find by that, compare all fields - GenericPK checkPK = checkValue.getPrimaryKey(); - GenericValue currentValue = delegator.findByPrimaryKey(checkPK); - - ModelEntity modelEntity = currentValue.getModelEntity(); - List nonpkFieldNameList = modelEntity.getNoPkFieldNames(); - Iterator nonpkFieldNameIter = nonpkFieldNameList.iterator(); - while (nonpkFieldNameIter.hasNext()) { - String nonpkFieldName = (String) nonpkFieldNameIter.next(); - Object checkField = checkValue.get(nonpkFieldName); - Object currentField = currentValue.get(nonpkFieldName); - - if (checkField != null && !checkField.equals(currentField)) { - StringBuffer matchError = new StringBuffer(); - matchError.append("Field [" + modelEntity.getEntityName() + "." + nonpkFieldName + "] did not match; file value [" + checkField + "], db value [" + currentField + "] pk [" + checkPK + "]"); - errorMessages.add(matchError.toString()); - } - } - + checkSingleValue(checkValue, delegator, errorMessages); rowsChecked++; } } catch (GenericEntityException e) { @@ -88,5 +69,44 @@ } return rowsChecked; + } + + public static void checkValueList(List valueList, GenericDelegator delegator, List errorMessages) throws GenericEntityException { + if (valueList == null) return; + + Iterator valueIter = valueList.iterator(); + while (valueIter.hasNext()) { + GenericValue checkValue = (GenericValue) valueIter.next(); + checkSingleValue(checkValue, delegator, errorMessages); + } + } + + public static void checkSingleValue(GenericValue checkValue, GenericDelegator delegator, List errorMessages) throws GenericEntityException { + // to check get the PK, find by that, compare all fields + GenericPK checkPK = checkValue.getPrimaryKey(); + GenericValue currentValue = delegator.findByPrimaryKey(checkPK); + if (currentValue == null) { + errorMessages.add("Entity [" + checkPK.getEntityName() + "] record not found for pk: " + checkPK); + } + + ModelEntity modelEntity = currentValue.getModelEntity(); + List nonpkFieldNameList = modelEntity.getNoPkFieldNames(); + Iterator nonpkFieldNameIter = nonpkFieldNameList.iterator(); + while (nonpkFieldNameIter.hasNext()) { + String nonpkFieldName = (String) nonpkFieldNameIter.next(); + // skip the fields the entity engine maintains + if (ModelEntity.CREATE_STAMP_FIELD.equals(nonpkFieldName) || ModelEntity.CREATE_STAMP_TX_FIELD.equals(nonpkFieldName) || + ModelEntity.STAMP_FIELD.equals(nonpkFieldName) || ModelEntity.STAMP_TX_FIELD.equals(nonpkFieldName)) { + continue; + } + + Object checkField = checkValue.get(nonpkFieldName); + Object currentField = currentValue.get(nonpkFieldName); + + if (checkField != null && !checkField.equals(currentField)) { + errorMessages.add("Field [" + modelEntity.getEntityName() + "." + nonpkFieldName + + "] did not match; file value [" + checkField + "], db value [" + currentField + "] pk [" + checkPK + "]"); + } + } } } Modified: ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntitySaxReader.java URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntitySaxReader.java?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntitySaxReader.java (original) +++ ofbiz/trunk/framework/entity/src/org/ofbiz/entity/util/EntitySaxReader.java Fri Aug 10 04:47:37 2007 @@ -37,6 +37,7 @@ import freemarker.template.TemplateHashModel; import javolution.text.CharArray; import javolution.text.Text; +import javolution.util.FastList; import javolution.util.FastMap; import javolution.xml.sax.XMLReaderImpl; import javolution.xml.sax.Attributes; @@ -81,8 +82,10 @@ protected boolean useTryInsertMethod = false; protected boolean maintainTxStamps = false; protected boolean createDummyFks = false; + protected boolean checkDataOnly = false; protected boolean doCacheClear = true; protected boolean disableEeca = false; + protected List messageList = null; protected List valuesToWrite = new ArrayList(valuesPerWrite); @@ -151,6 +154,14 @@ this.createDummyFks = createDummyFks; } + public boolean getCheckDataOnly() { + return this.checkDataOnly; + } + + public void setCheckDataOnly(boolean checkDataOnly) { + this.checkDataOnly = checkDataOnly; + } + public boolean getDoCacheClear() { return this.doCacheClear; } @@ -162,6 +173,17 @@ public boolean getDisableEeca() { return this.disableEeca; } + + public List getMessageList() { + if (this.checkDataOnly && this.messageList == null) { + messageList = FastList.newInstance(); + } + return this.messageList; + } + + public void setMessageList(List messageList) { + this.messageList = messageList; + } public void setDisableEeca(boolean disableEeca) { this.disableEeca = disableEeca; @@ -254,7 +276,11 @@ } protected void writeValues(List valuesToWrite) throws GenericEntityException { - delegator.storeAll(valuesToWrite, doCacheClear, createDummyFks); + if (this.checkDataOnly) { + EntityDataAssert.checkValueList(valuesToWrite, delegator, this.getMessageList()); + } else { + delegator.storeAll(valuesToWrite, doCacheClear, createDummyFks); + } } public void characters(char[] values, int offset, int count) throws org.xml.sax.SAXException { @@ -374,7 +400,7 @@ } try { - if (useTryInsertMethod) { + if (this.useTryInsertMethod && !this.checkDataOnly) { // this technique is faster for data sets where most, if not all, values do not already exist in the database try { currentValue.create(); Modified: ofbiz/trunk/framework/webtools/config/WebtoolsUiLabels.properties URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/config/WebtoolsUiLabels.properties?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/config/WebtoolsUiLabels.properties (original) +++ ofbiz/trunk/framework/webtools/config/WebtoolsUiLabels.properties Fri Aug 10 04:47:37 2007 @@ -68,6 +68,7 @@ WebtoolsCertsSerialNum=Serial Number WebtoolsCertsX509=x.509 Certificates WebtoolsCheckAll=Check All +WebtoolsCheckDataOnly=Check Data Only (nothing changed in database) WebtoolsCheckUpdateDatabase=Check/Update Database WebtoolsChildPeriods=Child Periods WebtoolsClearAllCaches=Clear All Caches Modified: ofbiz/trunk/framework/webtools/servicedef/services.xml URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/servicedef/services.xml?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/servicedef/services.xml (original) +++ ofbiz/trunk/framework/webtools/servicedef/services.xml Fri Aug 10 04:47:37 2007 @@ -35,6 +35,7 @@ <attribute name="maintainTimeStamps" type="String" mode="IN" optional="true"/> <attribute name="txTimeout" type="Integer" mode="IN" optional="true"/> <attribute name="createDummyFks" type="String" mode="IN" optional="true"/> + <attribute name="checkDataOnly" type="String" mode="IN" optional="true"/> <attribute name="rowProcessed" type="Long" mode="OUT" optional="false"/> </service> <service name="entityImport" engine="java" @@ -47,6 +48,7 @@ <attribute name="mostlyInserts" type="String" mode="IN" optional="true"/> <attribute name="maintainTimeStamps" type="String" mode="IN" optional="true"/> <attribute name="createDummyFks" type="String" mode="IN" optional="true"/> + <attribute name="checkDataOnly" type="String" mode="IN" optional="true"/> <attribute name="txTimeout" type="Integer" mode="IN" optional="true"/> <attribute name="messages" type="List" mode="OUT" optional="false"/> </service> @@ -57,6 +59,7 @@ <attribute name="mostlyInserts" type="String" mode="IN" optional="true"/> <attribute name="maintainTimeStamps" type="String" mode="IN" optional="true"/> <attribute name="createDummyFks" type="String" mode="IN" optional="true"/> + <attribute name="checkDataOnly" type="String" mode="IN" optional="true"/> <attribute name="deleteFiles" type="String" mode="IN" optional="true"/> <attribute name="txTimeout" type="Integer" mode="IN" optional="true"/> <attribute name="filePause" type="Long" mode="IN" optional="true"/> @@ -71,6 +74,7 @@ <attribute name="mostlyInserts" type="String" mode="IN" optional="true"/> <attribute name="maintainTimeStamps" type="String" mode="IN" optional="true"/> <attribute name="createDummyFks" type="String" mode="IN" optional="true"/> + <attribute name="checkDataOnly" type="String" mode="IN" optional="true"/> <attribute name="txTimeout" type="Integer" mode="IN" optional="true"/> <attribute name="messages" type="List" mode="OUT" optional="false"/> </service> Modified: ofbiz/trunk/framework/webtools/src/org/ofbiz/webtools/WebToolsServices.java URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/src/org/ofbiz/webtools/WebToolsServices.java?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/src/org/ofbiz/webtools/WebToolsServices.java (original) +++ ofbiz/trunk/framework/webtools/src/org/ofbiz/webtools/WebToolsServices.java Fri Aug 10 04:47:37 2007 @@ -42,6 +42,8 @@ import java.net.URL; import java.net.MalformedURLException; +import javax.xml.parsers.ParserConfigurationException; + import javolution.util.FastList; import org.ofbiz.base.util.Debug; @@ -54,6 +56,7 @@ import org.ofbiz.entity.GenericDelegator; import org.ofbiz.entity.GenericEntityException; import org.ofbiz.entity.GenericValue; +import org.ofbiz.entity.util.EntityDataAssert; import org.ofbiz.entity.util.EntityDataLoader; import org.ofbiz.entity.util.EntityListIterator; import org.ofbiz.entity.util.EntitySaxReader; @@ -66,6 +69,8 @@ import org.ofbiz.service.ServiceUtil; import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + import freemarker.template.*; import freemarker.ext.dom.NodeModel; import freemarker.ext.beans.BeansWrapper; @@ -96,6 +101,7 @@ String mostlyInserts = (String)context.get("mostlyInserts"); String maintainTimeStamps = (String)context.get("maintainTimeStamps"); String createDummyFks = (String)context.get("createDummyFks"); + String checkDataOnly = (String) context.get("checkDataOnly"); Integer txTimeout = (Integer)context.get("txTimeout"); @@ -170,6 +176,7 @@ try{ Map inputMap = UtilMisc.toMap("mostlyInserts", mostlyInserts, "createDummyFks", createDummyFks, + "checkDataOnly", checkDataOnly, "maintainTimeStamps", maintainTimeStamps, "txTimeout", txTimeout, "userLogin", userLogin); @@ -212,11 +219,12 @@ List messages = FastList.newInstance(); - String path = (String)context.get("path"); - String mostlyInserts = (String)context.get("mostlyInserts"); - String maintainTimeStamps = (String)context.get("maintainTimeStamps"); - String createDummyFks = (String)context.get("createDummyFks"); - boolean deleteFiles = (String)context.get("deleteFiles") != null; + String path = (String) context.get("path"); + String mostlyInserts = (String) context.get("mostlyInserts"); + String maintainTimeStamps = (String) context.get("maintainTimeStamps"); + String createDummyFks = (String) context.get("createDummyFks"); + boolean deleteFiles = (String) context.get("deleteFiles") != null; + String checkDataOnly = (String) context.get("checkDataOnly"); Integer txTimeout = (Integer)context.get("txTimeout"); Long filePause = (Long)context.get("filePause"); @@ -253,6 +261,7 @@ while (filesItr.hasNext()) { Map parseEntityXmlFileArgs = UtilMisc.toMap("mostlyInserts", mostlyInserts, "createDummyFks", createDummyFks, + "checkDataOnly", checkDataOnly, "maintainTimeStamps", maintainTimeStamps, "txTimeout", txTimeout, "userLogin", userLogin); @@ -322,7 +331,8 @@ boolean useDummyFks = "true".equals((String) context.get("createDummyFks")); boolean maintainTxs = "true".equals((String) context.get("maintainTimeStamps")); boolean tryInserts = "true".equals((String) context.get("mostlyInserts")); - + boolean checkDataOnly = "true".equals((String) context.get("checkDataOnly")); + Integer txTimeoutInt = (Integer) context.get("txTimeout"); int txTimeout = txTimeoutInt != null ? txTimeoutInt.intValue() : -1; @@ -369,20 +379,34 @@ List infoMessages = new LinkedList(); int totalRowsChanged = 0; if (urlList != null && urlList.size() > 0) { - messages.add("=-=-=-=-=-=-= Doing a data load with the following files:"); + messages.add("=-=-=-=-=-=-= Doing a data " + (checkDataOnly ? "check" : "load") + " with the following files:"); Iterator urlIter = urlList.iterator(); while (urlIter.hasNext()) { URL dataUrl = (URL) urlIter.next(); messages.add(dataUrl.toExternalForm()); } - messages.add("=-=-=-=-=-=-= Starting the data load..."); + messages.add("=-=-=-=-=-=-= Starting the data " + (checkDataOnly ? "check" : "load") + "..."); urlIter = urlList.iterator(); while (urlIter.hasNext()) { URL dataUrl = (URL) urlIter.next(); try { - int rowsChanged = EntityDataLoader.loadData(dataUrl, helperName, delegator, errorMessages, txTimeout, useDummyFks, maintainTxs, tryInserts); + int rowsChanged = 0; + if (checkDataOnly) { + try { + errorMessages.add("Checking data in [" + dataUrl.toExternalForm() + "]"); + rowsChanged = EntityDataAssert.assertData(dataUrl, delegator, errorMessages); + } catch (SAXException e) { + errorMessages.add("Error checking data in [" + dataUrl.toExternalForm() + "]: " + e.toString()); + } catch (ParserConfigurationException e) { + errorMessages.add("Error checking data in [" + dataUrl.toExternalForm() + "]: " + e.toString()); + } catch (IOException e) { + errorMessages.add("Error checking data in [" + dataUrl.toExternalForm() + "]: " + e.toString()); + } + } else { + rowsChanged = EntityDataLoader.loadData(dataUrl, helperName, delegator, errorMessages, txTimeout, useDummyFks, maintainTxs, tryInserts); + } totalRowsChanged += rowsChanged; infoMessages.add(changedFormat.format(rowsChanged) + " of " + changedFormat.format(totalRowsChanged) + " from " + dataUrl.toExternalForm()); } catch (GenericEntityException e) { @@ -390,20 +414,20 @@ } } } else { - messages.add("=-=-=-=-=-=-= No data load files found."); + messages.add("=-=-=-=-=-=-= No data " + (checkDataOnly ? "check" : "load") + " files found."); } if (infoMessages.size() > 0) { - messages.add("=-=-=-=-=-=-= Here is a summary of the data load:"); + messages.add("=-=-=-=-=-=-= Here is a summary of the data " + (checkDataOnly ? "check" : "load") + ":"); messages.addAll(infoMessages); } if (errorMessages.size() > 0) { - messages.add("=-=-=-=-=-=-= The following errors occured in the data load:"); + messages.add("=-=-=-=-=-=-= The following errors occured in the data " + (checkDataOnly ? "check" : "load") + ":"); messages.addAll(errorMessages); } - messages.add("=-=-=-=-=-=-= Finished the data load with " + totalRowsChanged + " rows changed."); + messages.add("=-=-=-=-=-=-= Finished the data " + (checkDataOnly ? "check" : "load") + " with " + totalRowsChanged + " rows " + (checkDataOnly ? "checked" : "changed") + "."); Map resultMap = ServiceUtil.returnSuccess(); resultMap.put("messages", messages); @@ -428,6 +452,7 @@ boolean mostlyInserts = (String)context.get("mostlyInserts") != null; boolean maintainTimeStamps = (String)context.get("maintainTimeStamps") != null; boolean createDummyFks = (String)context.get("createDummyFks") != null; + boolean checkDataOnly = (String)context.get("checkDataOnly") != null; Integer txTimeout = (Integer)context.get("txTimeout"); if (txTimeout == null) { @@ -441,8 +466,9 @@ reader.setMaintainTxStamps(maintainTimeStamps); reader.setTransactionTimeout(txTimeout.intValue()); reader.setCreateDummyFks(createDummyFks); + reader.setCheckDataOnly(checkDataOnly); - long numberRead = (url != null? reader.parse(url): reader.parse(xmltext)); + long numberRead = (url != null ? reader.parse(url) : reader.parse(xmltext)); rowProcessed = new Long(numberRead); } catch (Exception ex){ return ServiceUtil.returnError("Error parsing entity xml file: " + ex.toString()); Modified: ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImport.ftl URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImport.ftl?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImport.ftl (original) +++ ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImport.ftl Fri Aug 10 04:47:37 2007 @@ -33,6 +33,7 @@ <input type="checkbox" name="mostlyInserts" <#if mostlyInserts?exists>"checked"</#if>/>${uiLabelMap.WebtoolsMostlyInserts}<br /> <input type="checkbox" name="maintainTimeStamps" <#if keepStamps?exists>"checked"</#if>/>${uiLabelMap.WebtoolsMaintainTimestamps}<br /> <input type="checkbox" name="createDummyFks" <#if createDummyFks?exists>"checked"</#if>/>${uiLabelMap.WebtoolsCreateDummyFks}<br /> + <input type="checkbox" name="checkDataOnly" <#if checkDataOnly?exists>"checked"</#if>/>${uiLabelMap.WebtoolsCheckDataOnly}<br /> ${uiLabelMap.WebtoolsTimeoutSeconds}:<input type="text" size="6" value="${txTimeoutStr?default("7200")}" name="txTimeout"/><br /> <div class="button-bar"><input type="submit" value="${uiLabelMap.WebtoolsImportFile}"/></div> </form> Modified: ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportDir.ftl URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportDir.ftl?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportDir.ftl (original) +++ ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportDir.ftl Fri Aug 10 04:47:37 2007 @@ -30,6 +30,7 @@ <input type="checkbox" name="maintainTimeStamps" <#if keepStamps?exists>checked="checked"</#if>/>${uiLabelMap.WebtoolsMaintainTimestamps}<br /> <input type="checkbox" name="createDummyFks" <#if createDummyFks?exists>checked="checked"</#if>/>${uiLabelMap.WebtoolsCreateDummyFks}<br /> <input type="checkbox" name="deleteFiles" <#if (deleteFiles?exists)>checked="checked"</#if>/>${uiLabelMap.WebtoolsDeleteFiles}<br /> + <input type="checkbox" name="checkDataOnly" <#if checkDataOnly?exists>"checked"</#if>/>${uiLabelMap.WebtoolsCheckDataOnly}<br /> ${uiLabelMap.WebtoolsTimeoutSeconds}:<input type="text" size="6" value="${txTimeoutStr?default("7200")}" name="txTimeout"/><br /> ${uiLabelMap.WebtoolsPause}:<input type="text" size="6" value="${filePauseStr?default("0")}" name="filePause"/><br /> <div class="button-bar"><input type="submit" value="${uiLabelMap.WebtoolsImportFile}"/></div> Modified: ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportReaders.ftl URL: http://svn.apache.org/viewvc/ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportReaders.ftl?view=diff&rev=564565&r1=564564&r2=564565 ============================================================================== --- ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportReaders.ftl (original) +++ ofbiz/trunk/framework/webtools/webapp/webtools/entity/EntityImportReaders.ftl Fri Aug 10 04:47:37 2007 @@ -26,9 +26,10 @@ <form method="post" action="<@ofbizUrl>entityImportReaders</@ofbizUrl>"> Enter Readers (comma separated, no spaces; from entityengine.xml and ofbiz-component.xml files; common ones include seed,ext,demo):</div> <input type="text" size="60" name="readers" value="${readers?default("seed")}"/><br /> - <input type="checkbox" name="mostlyInserts" <#if mostlyInserts?exists>"checked"</#if>/>${uiLabelMap.WebtoolsMostlyInserts}<br /> - <input type="checkbox" name="maintainTimeStamps" <#if keepStamps?exists>"checked"</#if>/>${uiLabelMap.WebtoolsMaintainTimestamps}<br /> - <input type="checkbox" name="createDummyFks" <#if createDummyFks?exists>"checked"</#if>/>${uiLabelMap.WebtoolsCreateDummyFks}<br /> + <input type="checkbox" name="mostlyInserts" <#if mostlyInserts?exists>"checked"</#if> value="true"/>${uiLabelMap.WebtoolsMostlyInserts}<br /> + <input type="checkbox" name="maintainTimeStamps" <#if keepStamps?exists>"checked"</#if> value="true"/>${uiLabelMap.WebtoolsMaintainTimestamps}<br /> + <input type="checkbox" name="createDummyFks" <#if createDummyFks?exists>"checked"</#if> value="true"/>${uiLabelMap.WebtoolsCreateDummyFks}<br /> + <input type="checkbox" name="checkDataOnly" <#if checkDataOnly?exists>"checked"</#if> value="true"/>${uiLabelMap.WebtoolsCheckDataOnly}<br /> ${uiLabelMap.WebtoolsTimeoutSeconds}:<input type="text" size="6" value="${txTimeoutStr?default("7200")}" name="txTimeout"/><br /> <div class="button-bar"><input type="submit" value="${uiLabelMap.WebtoolsImport}"/></div> </form> |
Free forum by Nabble | Edit this page |