From 4409415d5501fe954c61a053b726a6940c6962d2 Mon Sep 17 00:00:00 2001 From: Carlos Ruiz Date: Wed, 8 May 2024 14:16:12 +0200 Subject: [PATCH] IDEMPIERE-6123 Query in search window causing slowness and load spikes in the database (FHCA-5356) (#2354) * IDEMPIERE-6123 Query in search window causing slowness and load spikes in the database (FHCA-5356) - make also configurable the timeout and number of records from windows - improve timing of reports avoiding unncessary load of array when not jasper * - implement suggestions from Heng Sin --- .../oracle/202405080101_IDEMPIERE-6123.sql | 22 +++++++++++++++ .../202405080101_IDEMPIERE-6123.sql | 19 +++++++++++++ .../org/compiere/db/AdempiereDatabase.java | 2 +- .../src/org/compiere/model/MAcctSchema.java | 2 +- .../src/org/compiere/model/MLookup.java | 2 ++ .../compiere/model/MProductCategoryAcct.java | 2 +- .../src/org/compiere/model/MSysConfig.java | 4 ++- .../src/org/compiere/print/DataEngine.java | 26 +++++++++++++++-- .../webui/panel/action/ReportAction.java | 28 ++++++++++--------- 9 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 migration/iD11/oracle/202405080101_IDEMPIERE-6123.sql create mode 100644 migration/iD11/postgresql/202405080101_IDEMPIERE-6123.sql diff --git a/migration/iD11/oracle/202405080101_IDEMPIERE-6123.sql b/migration/iD11/oracle/202405080101_IDEMPIERE-6123.sql new file mode 100644 index 0000000000..ffd792aa44 --- /dev/null +++ b/migration/iD11/oracle/202405080101_IDEMPIERE-6123.sql @@ -0,0 +1,22 @@ +-- IDEMPIERE-6123 Query in search window causing slowness and load spikes in the database (FHCA-5356) +SELECT register_migration_script('202405080101_IDEMPIERE-6123.sql') FROM dual; + +SET SQLBLANKLINES ON +SET DEFINE OFF + +-- May 8, 2024, 1:01:16 AM CEST +INSERT INTO AD_SysConfig (AD_SysConfig_ID,AD_Client_ID,AD_Org_ID,Created,Updated,CreatedBy,UpdatedBy,IsActive,Name,Value,Description,EntityType,ConfigurationLevel,AD_SysConfig_UU) VALUES (200247,0,0,TO_TIMESTAMP('2024-05-08 01:01:16','YYYY-MM-DD HH24:MI:SS'),TO_TIMESTAMP('2024-05-08 01:01:16','YYYY-MM-DD HH24:MI:SS'),100,100,'Y','REPORT_LOAD_TIMEOUT_IN_SECONDS','120','Timeout in seconds when loading a report','D','C','14e838b1-c25c-400e-b39c-61da9bf92099') +; + +-- May 8, 2024, 1:01:42 AM CEST +INSERT INTO AD_SysConfig (AD_SysConfig_ID,AD_Client_ID,AD_Org_ID,Created,Updated,CreatedBy,UpdatedBy,IsActive,Name,Value,Description,EntityType,ConfigurationLevel,AD_SysConfig_UU) VALUES (200248,0,0,TO_TIMESTAMP('2024-05-08 01:01:41','YYYY-MM-DD HH24:MI:SS'),TO_TIMESTAMP('2024-05-08 01:01:41','YYYY-MM-DD HH24:MI:SS'),100,100,'Y','GLOBAL_MAX_REPORT_RECORDS','100000','Max number of records allowed in a report','D','C','7030640a-1aa7-4ac7-a894-b4fe0dfde530') +; + +-- May 8, 2024, 1:06:19 AM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','The data query for the report took too much time to execute (over {0} seconds) exceeding the allowed limit',0,0,'Y',TO_TIMESTAMP('2024-05-08 01:06:18','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-05-08 01:06:18','YYYY-MM-DD HH24:MI:SS'),100,200893,'ReportQueryTimeout','D','5f17f55f-adbe-4d97-bf83-9447983b4946') +; + +-- May 8, 2024, 1:07:10 AM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','The report data exceeds the maximum limit of {0} rows',0,0,'Y',TO_TIMESTAMP('2024-05-08 01:07:09','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-05-08 01:07:09','YYYY-MM-DD HH24:MI:SS'),100,200894,'ReportMaxRowsReached','D','a4b55c31-0df0-4302-a62a-91cb7e79be0d') +; + diff --git a/migration/iD11/postgresql/202405080101_IDEMPIERE-6123.sql b/migration/iD11/postgresql/202405080101_IDEMPIERE-6123.sql new file mode 100644 index 0000000000..be168edcfe --- /dev/null +++ b/migration/iD11/postgresql/202405080101_IDEMPIERE-6123.sql @@ -0,0 +1,19 @@ +-- IDEMPIERE-6123 Query in search window causing slowness and load spikes in the database (FHCA-5356) +SELECT register_migration_script('202405080101_IDEMPIERE-6123.sql') FROM dual; + +-- May 8, 2024, 1:01:16 AM CEST +INSERT INTO AD_SysConfig (AD_SysConfig_ID,AD_Client_ID,AD_Org_ID,Created,Updated,CreatedBy,UpdatedBy,IsActive,Name,Value,Description,EntityType,ConfigurationLevel,AD_SysConfig_UU) VALUES (200247,0,0,TO_TIMESTAMP('2024-05-08 01:01:16','YYYY-MM-DD HH24:MI:SS'),TO_TIMESTAMP('2024-05-08 01:01:16','YYYY-MM-DD HH24:MI:SS'),100,100,'Y','REPORT_LOAD_TIMEOUT_IN_SECONDS','120','Timeout in seconds when loading a report','D','C','14e838b1-c25c-400e-b39c-61da9bf92099') +; + +-- May 8, 2024, 1:01:42 AM CEST +INSERT INTO AD_SysConfig (AD_SysConfig_ID,AD_Client_ID,AD_Org_ID,Created,Updated,CreatedBy,UpdatedBy,IsActive,Name,Value,Description,EntityType,ConfigurationLevel,AD_SysConfig_UU) VALUES (200248,0,0,TO_TIMESTAMP('2024-05-08 01:01:41','YYYY-MM-DD HH24:MI:SS'),TO_TIMESTAMP('2024-05-08 01:01:41','YYYY-MM-DD HH24:MI:SS'),100,100,'Y','GLOBAL_MAX_REPORT_RECORDS','100000','Max number of records allowed in a report','D','C','7030640a-1aa7-4ac7-a894-b4fe0dfde530') +; + +-- May 8, 2024, 1:06:19 AM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','The data query for the report took too much time to execute (over {0} seconds) exceeding the allowed limit',0,0,'Y',TO_TIMESTAMP('2024-05-08 01:06:18','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-05-08 01:06:18','YYYY-MM-DD HH24:MI:SS'),100,200893,'ReportQueryTimeout','D','5f17f55f-adbe-4d97-bf83-9447983b4946') +; + +-- May 8, 2024, 1:07:10 AM CEST +INSERT INTO AD_Message (MsgType,MsgText,AD_Client_ID,AD_Org_ID,IsActive,Created,CreatedBy,Updated,UpdatedBy,AD_Message_ID,Value,EntityType,AD_Message_UU) VALUES ('I','The report data exceeds the maximum limit of {0} rows',0,0,'Y',TO_TIMESTAMP('2024-05-08 01:07:09','YYYY-MM-DD HH24:MI:SS'),100,TO_TIMESTAMP('2024-05-08 01:07:09','YYYY-MM-DD HH24:MI:SS'),100,200894,'ReportMaxRowsReached','D','a4b55c31-0df0-4302-a62a-91cb7e79be0d') +; + diff --git a/org.adempiere.base/src/org/compiere/db/AdempiereDatabase.java b/org.adempiere.base/src/org/compiere/db/AdempiereDatabase.java index 9b4e0afc18..84d1034f25 100644 --- a/org.adempiere.base/src/org/compiere/db/AdempiereDatabase.java +++ b/org.adempiere.base/src/org/compiere/db/AdempiereDatabase.java @@ -312,7 +312,7 @@ public interface AdempiereDatabase public boolean isPagingSupported(); /** - * modify sql to return a subset of the query result + * modify sql to return a subset of the query result. use 1 base index for start and end parameter * @param sql * @param start * @param end diff --git a/org.adempiere.base/src/org/compiere/model/MAcctSchema.java b/org.adempiere.base/src/org/compiere/model/MAcctSchema.java index 8976261c95..7ec3cf820f 100644 --- a/org.adempiere.base/src/org/compiere/model/MAcctSchema.java +++ b/org.adempiere.base/src/org/compiere/model/MAcctSchema.java @@ -744,7 +744,7 @@ public class MAcctSchema extends X_C_AcctSchema implements ImmutablePOSupport StringBuilder sql = new StringBuilder("SELECT DISTINCT p.Value FROM M_Product p JOIN M_CostDetail d ON p.M_Product_ID=d.M_Product_ID"); sql.append(" JOIN M_Product_Category_Acct pc ON p.M_Product_Category_ID=pc.M_Product_Category_ID AND d.C_AcctSchema_ID=pc.C_AcctSchema_ID"); sql.append(" WHERE p.IsActive='Y' AND pc.IsActive='Y' AND pc.CostingLevel IS NULL AND d.C_AcctSchema_ID=?"); - String query = DB.getDatabase().addPagingSQL(sql.toString(), 0, 50); + String query = DB.getDatabase().addPagingSQL(sql.toString(), 1, 50); List> list = DB.getSQLArrayObjectsEx(get_TrxName(), query, getC_AcctSchema_ID()); if (list != null) { for(List entry : list) { diff --git a/org.adempiere.base/src/org/compiere/model/MLookup.java b/org.adempiere.base/src/org/compiere/model/MLookup.java index 756cc922fe..4747f754f0 100644 --- a/org.adempiere.base/src/org/compiere/model/MLookup.java +++ b/org.adempiere.base/src/org/compiere/model/MLookup.java @@ -1230,6 +1230,8 @@ public final class MLookup extends Lookup implements Serializable // SELECT Key, Value, Name, IsActive FROM ... String sqlFirstRows = DB.getDatabase().addPagingSQL(sql.toString(), 1, MAX_ROWS+1); pstmt = DB.prepareStatement(sqlFirstRows, null); + if (! DB.getDatabase().isPagingSupported()) + pstmt.setMaxRows(MAX_ROWS+1); int timeout = MSysConfig.getIntValue(MSysConfig.GRIDTABLE_LOAD_TIMEOUT_IN_SECONDS, GridTable.DEFAULT_GRIDTABLE_LOAD_TIMEOUT_IN_SECONDS, Env.getAD_Client_ID(Env.getCtx())); if (timeout > 0) pstmt.setQueryTimeout(timeout); diff --git a/org.adempiere.base/src/org/compiere/model/MProductCategoryAcct.java b/org.adempiere.base/src/org/compiere/model/MProductCategoryAcct.java index 63497f8d39..653e9c41a9 100644 --- a/org.adempiere.base/src/org/compiere/model/MProductCategoryAcct.java +++ b/org.adempiere.base/src/org/compiere/model/MProductCategoryAcct.java @@ -236,7 +236,7 @@ public class MProductCategoryAcct extends X_M_Product_Category_Acct implements I StringBuilder products = new StringBuilder(); StringBuilder sql = new StringBuilder("SELECT DISTINCT p.Value FROM M_Product p JOIN M_CostDetail d ON p.M_Product_ID=d.M_Product_ID"); sql.append(" WHERE p.IsActive='Y' AND p.M_Product_Category_ID=? AND d.C_AcctSchema_ID=?"); - String query = DB.getDatabase().addPagingSQL(sql.toString(), 0, 50); + String query = DB.getDatabase().addPagingSQL(sql.toString(), 1, 50); List> list = DB.getSQLArrayObjectsEx(get_TrxName(), query, getM_Product_Category_ID(), getC_AcctSchema_ID()); if (list != null) { for(List entry : list) { diff --git a/org.adempiere.base/src/org/compiere/model/MSysConfig.java b/org.adempiere.base/src/org/compiere/model/MSysConfig.java index eabb6da52d..9dea5550dd 100644 --- a/org.adempiere.base/src/org/compiere/model/MSysConfig.java +++ b/org.adempiere.base/src/org/compiere/model/MSysConfig.java @@ -46,7 +46,7 @@ public class MSysConfig extends X_AD_SysConfig /** * */ - private static final long serialVersionUID = 8121897973805635995L; + private static final long serialVersionUID = 5739824752288579881L; /** Constant for Predefine System Configuration Names (in alphabetical order) */ @@ -120,6 +120,7 @@ public class MSysConfig extends X_AD_SysConfig public static final String FORM_SQL_QUERY_MAX_RECORDS = "FORM_SQL_QUERY_MAX_RECORDS"; public static final String FORM_SQL_QUERY_TIMEOUT_IN_SECONDS = "FORM_SQL_QUERY_TIMEOUT_IN_SECONDS"; public static final String GLOBAL_MAX_QUERY_RECORDS = "GLOBAL_MAX_QUERY_RECORDS"; + public static final String GLOBAL_MAX_REPORT_RECORDS = "GLOBAL_MAX_REPORT_RECORDS"; public static final String GRIDTABLE_LOAD_TIMEOUT_IN_SECONDS = "GRIDTABLE_LOAD_TIMEOUT_IN_SECONDS"; public static final String GRIDTABLE_INITIAL_COUNT_TIMEOUT_IN_SECONDS = "GRIDTABLE_INITIAL_COUNT_TIMEOUT_IN_SECONDS"; public static final String HTML_REPORT_MINIFY = "HTML_REPORT_MINIFY"; @@ -185,6 +186,7 @@ public class MSysConfig extends X_AD_SysConfig public static final String REAL_TIME_POS = "REAL_TIME_POS"; public static final String RecentItems_MaxSaved = "RecentItems_MaxSaved"; public static final String RecentItems_MaxShown = "RecentItems_MaxShown"; + public static final String REPORT_LOAD_TIMEOUT_IN_SECONDS = "REPORT_LOAD_TIMEOUT_IN_SECONDS"; public static final String REPORT_SWAP_MAX_ROWS = "REPORT_SWAP_MAX_ROWS"; public static final String SHIPPING_DEFAULT_WEIGHT_PER_PACKAGE = "SHIPPING_DEFAULT_WEIGHT_PER_PACKAGE"; public static final String STANDARD_REPORT_FOOTER_TRADEMARK_TEXT = "STANDARD_REPORT_FOOTER_TRADEMARK_TEXT"; diff --git a/org.adempiere.base/src/org/compiere/print/DataEngine.java b/org.adempiere.base/src/org/compiere/print/DataEngine.java index 416ea7c7d4..50a2569590 100644 --- a/org.adempiere.base/src/org/compiere/print/DataEngine.java +++ b/org.adempiere.base/src/org/compiere/print/DataEngine.java @@ -41,6 +41,7 @@ import org.compiere.model.MLookupFactory; import org.compiere.model.MQuery; import org.compiere.model.MReportView; import org.compiere.model.MRole; +import org.compiere.model.MSysConfig; import org.compiere.model.MTable; import org.compiere.model.SystemIDs; import org.compiere.util.CLogMgt; @@ -142,6 +143,10 @@ public class DataEngine private Map m_summarized = new HashMap(); + public static final int DEFAULT_REPORT_LOAD_TIMEOUT_IN_SECONDS = 120; + + public static final int DEFAULT_GLOBAL_MAX_REPORT_RECORDS = 100000; + /************************************************************************** * Load Data * @@ -927,11 +932,20 @@ public class DataEngine int reportLineID = 0; ArrayList scriptColumns = new ArrayList(); // + int timeout = MSysConfig.getIntValue(MSysConfig.REPORT_LOAD_TIMEOUT_IN_SECONDS, DEFAULT_REPORT_LOAD_TIMEOUT_IN_SECONDS, Env.getAD_Client_ID(Env.getCtx())); PreparedStatement pstmt = null; ResultSet rs = null; + String sql = pd.getSQL(); try { - pstmt = DB.prepareNormalReadReplicaStatement(pd.getSQL(), m_trxName); + int maxRows = MSysConfig.getIntValue(MSysConfig.GLOBAL_MAX_REPORT_RECORDS, DEFAULT_GLOBAL_MAX_REPORT_RECORDS, Env.getAD_Client_ID(Env.getCtx())); + if (maxRows > 0 && DB.getDatabase().isPagingSupported()) + sql = DB.getDatabase().addPagingSQL(sql, 1, maxRows+1); + pstmt = DB.prepareNormalReadReplicaStatement(sql, m_trxName); + if (maxRows > 0 && ! DB.getDatabase().isPagingSupported()) + pstmt.setMaxRows(maxRows+1); + if (timeout > 0) + pstmt.setQueryTimeout(timeout); rs = pstmt.executeQuery(); boolean isExistsT_Report_PA_ReportLine_ID = false; @@ -948,9 +962,13 @@ public class DataEngine } } + int cnt = 0; // Row Loop while (rs.next()) { + cnt++; + if (maxRows > 0 && cnt > maxRows) + throw new AdempiereException(Msg.getMsg(Env.getCtx(), "ReportMaxRowsReached", new Object[] {maxRows})); if (hasLevelNo) { levelNo = rs.getInt("LevelNo"); @@ -1184,7 +1202,9 @@ public class DataEngine } catch (SQLException e) { - log.log(Level.SEVERE, pdc + " - " + e.getMessage() + "\nSQL=" + pd.getSQL()); + if (DB.getDatabase().isQueryTimeout(e)) + throw new AdempiereException(Msg.getMsg(Env.getCtx(), "ReportQueryTimeout", new Object[] {timeout})); + log.log(Level.SEVERE, pdc + " - " + e.getMessage() + "\nSQL=" + sql); throw new AdempiereException(e); } finally @@ -1303,7 +1323,7 @@ public class DataEngine { if (CLogMgt.isLevelFiner()) log.finer("NO Rows - ms=" + (System.currentTimeMillis()-m_startTime) - + " - " + pd.getSQL()); + + " - " + sql); else log.info("NO Rows - ms=" + (System.currentTimeMillis()-m_startTime)); } diff --git a/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/ReportAction.java b/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/ReportAction.java index 42c00a9a87..4479a2711e 100644 --- a/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/ReportAction.java +++ b/org.adempiere.ui.zk/WEB-INF/src/org/adempiere/webui/panel/action/ReportAction.java @@ -291,8 +291,8 @@ public class ReportAction implements EventListener boolean currentRowOnly = chkCurrentRowOnly.isChecked(); int Record_ID = 0; String Record_UU = null; - List RecordIDs = null; - List RecordUUs = null; + List jasperRecordIDs = null; + List jasperRecordUUs = null; MQuery query = new MQuery(gridTab.getTableName()); MTable table = MTable.get(gridTab.getAD_Table_ID()); StringBuilder whereClause = new StringBuilder(""); @@ -309,15 +309,17 @@ public class ReportAction implements EventListener else { whereClause.append(gridTab.getTableModel().getSelectWhereClause()); - if (table.isUUIDKeyTable()) { - RecordUUs = new ArrayList(); - for(int i = 0; i < gridTab.getRowCount(); i++) { - RecordUUs.add(gridTab.getKeyUUID(i)); - } - } else { - RecordIDs = new ArrayList(); - for(int i = 0; i < gridTab.getRowCount(); i++) { - RecordIDs.add(gridTab.getKeyID(i)); + if (pf != null && pf.getJasperProcess_ID() > 0) { + if (table.isUUIDKeyTable()) { + jasperRecordUUs = new ArrayList(); + for(int i = 0; i < gridTab.getRowCount(); i++) { + jasperRecordUUs.add(gridTab.getKeyUUID(i)); + } + } else { + jasperRecordIDs = new ArrayList(); + for(int i = 0; i < gridTab.getRowCount(); i++) { + jasperRecordIDs.add(gridTab.getKeyID(i)); + } } } } @@ -358,8 +360,8 @@ public class ReportAction implements EventListener { // It's a report using the JasperReports engine ProcessInfo pi = new ProcessInfo ("", pf.getJasperProcess_ID(), pf.getAD_Table_ID(), Record_ID, Record_UU); - pi.setRecord_IDs(RecordIDs); - pi.setRecord_UUs(RecordUUs); + pi.setRecord_IDs(jasperRecordIDs); + pi.setRecord_UUs(jasperRecordUUs); //pi.setIsBatch(true); if (export)