From 1af27e938542b18fe41b1e0c807aae5bc52dc802 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Tue, 27 Oct 2026 10:59:26 +0100 Subject: [PATCH 01/13] SQLRDD: Append() without MAX(recno), report a failed insert Append() asked the server for MAX(recno) + 1 as the key of the new row, but with a recno column the insert itself returns the new record number, which then replaced it. That was one round trip per Append() for nothing; the returned record number now also updates LastRec(). A failed insert (e.g. a CHECK constraint) was rolled back and reported as success: the work area stood on a row that did not exist on the server and the following field changes went nowhere. The error is now registered as an EG_WRITE error and Append() returns FALSE, as a failed append does under DBF. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 15 ++++++++++++--- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 6 +++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 61ef6ab3c5..54ea612e7c 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -280,7 +280,9 @@ partial class SQLRDD inherit Workarea var lResult := SELF:GoCold() if lResult self:_eofPending := false - var key := (dword) self:_builder:GetNextKey() + // With a recno column the insert below returns the new record number; asking the server + // for MAX(recno) first only cost a round trip per Append(). + var key := iif(self:_recnoColumNo != -1, 0U, (dword) self:_builder:GetNextKey()) var row := self:DataTable:NewRow() self:DataTable:Rows:Add(row) // The new row does not belong at the end of the buffered slice of the order. @@ -296,9 +298,16 @@ partial class SQLRDD inherit Workarea row[c] := SELF:_HandleNullDate(values[c:Ordinal],c) endif next - self:_ExecuteInsertStatement(row) + if !self:_ExecuteInsertStatement(row) + // the error is reported, stay where we were + self:DataTable:Rows:Remove(row) + return false + endif row:AcceptChanges() - _updatedRecNos:Add((int)row[self:_recnoColumNo]) + if self:_recnoColumNo != -1 + key := Convert.ToUInt32(row[self:_recnoColumNo]) + endif + _updatedRecNos:Add((int) key) SELF:RowNumber := SELF:DataTable:Rows:Count self:_serverReccount += 1 if self:_tableReccountValid diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 336b6fabc5..26cabf024f 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -425,8 +425,12 @@ partial class SQLRDD _command:ExecuteNonQuery() endif _command:Connection:CommitTrans() - catch + catch e as Exception + // Reporting success here made a failed insert look like a new record whose changes + // then went nowhere. _command:Connection:RollBackTrans() + self:_dbfError(ERDD.WRITE, XSharp.Gencode.EG_WRITE, "SqlRDD:Append", e:Message, FALSE) + return false end try return true end method From f42092477c7fe459e5f5d4a30300a61a426be17d Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Tue, 27 Oct 2026 10:59:48 +0100 Subject: [PATCH 02/13] SQLRDD: EOF positioning like ADS without extra queries Code that saves and restores the position of a work area (order and recno, e.g. around a lookup) does SetOrder() and GoTo(saved recno). At EOF that recno is LastRec()+1, and in an ERP order copy this ran thousands of times on empty tables (tax and dimension tables). Each time SQLRDD sent four queries (single record, keyset window, ROW_NUMBER(), single record again) and counted the order to compute BOF. - GoTo() to a record that does not exist goes to EOF as soon as the lookup by its recno finds nothing. Before, it stayed on the previous record. Like ADS it sets BOF as well, also for GoTo(0). - A backward Skip() at EOF and BOF lands on the first record with BOF, as with ADS. - SetOrder() to the active order at EOF stays there instead of GoTo(). - An order change at EOF keeps BOF as it was (ADS), so the order is no longer counted for it. - At a pending EOF (phantom record with a buffer that does not end at the last record) Skip(1) stays at EOF also the second time; before, it read on behind the buffer. - Found is reset by GoTo(), GoTop(), GoBottom() and Skip(), as in VO. - GoToId() no longer recomputes EOF/BOF after GoTo(), which cleared BOF at the phantom record. Checked against AXDBFCDX on tables with 1, 338, 1,025 and 1,245 rows (GoTo past the end, GoTo(0), order changes and seeks at EOF, backward skips): identical EOF/BOF/RecNo except LastRec where the data differ. Empty tables differ in one ADS quirk (ADS clears EOF after Skip(-1)). Statements for that sequence: 44-61 -> 21. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 34 ++++++++++++++--- .../XSharp.SQLRdd/RDD/SQLRDD-Orders.prg | 11 ++++-- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 38 ++++++++++++------- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 54ea612e7c..aada7e06eb 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -664,6 +664,8 @@ partial class SQLRDD inherit Workarea if !self:_ForceOpen() return false endif + // VO resets FOUND to FALSE after a record movement + SELF:_Found := FALSE // Reloading the buffer below discards pending FieldPut()s, so write them first (DBF does too). SELF:GoCold() SELF:_outsideOrder := FALSE @@ -692,6 +694,7 @@ partial class SQLRDD inherit Workarea if !self:_ForceOpen() return false endif + SELF:_Found := FALSE // Reloading the buffer below discards pending FieldPut()s, so write them first (DBF does too). SELF:GoCold() SELF:_outsideOrder := FALSE @@ -741,11 +744,12 @@ partial class SQLRDD inherit Workarea SELF:GoCold() IF nToSkip != 0 .AND. SELF:_eofPending // On the phantom record after GoTo(0), with a buffer that does not end at the last - // record: moving forward stays at EOF, moving backward starts from the last record. - SELF:_eofPending := FALSE + // record: moving forward stays at EOF (still pending, so a second Skip(1) does not read on + // behind the buffer), moving backward starts from the last record. IF nToSkip > 0 RETURN TRUE ENDIF + SELF:_eofPending := FALSE isOK := SELF:GoBottom() IF isOK .AND. nToSkip < -1 isOK := SELF:_SkipInBuffer(nToSkip + 1) @@ -768,7 +772,8 @@ partial class SQLRDD inherit Workarea // phantom row is only built once a SELECT actually loaded a schema - see the // DataTable setter) - same case GetValue() already guards against above. if self:CurrentRow == null .or. oRec != self:CurrentRow[self:_recnoColumNo] - result := SELF:GoTo( (DWORD) nRec ) + // GoTo() sets EOF/BOF itself (at the phantom record BOF as ADS does) + return SELF:GoTo( (DWORD) nRec ) endif CATCH ex AS Exception SELF:_dbfError(ex, Subcodes.EDB_GOTO,Gencode.EG_DATATYPE, "SQLRDD.GoToId",FALSE) @@ -790,7 +795,9 @@ partial class SQLRDD inherit Workarea IF nRec == 0 // DBF semantics: GoTo(0) positions on the phantom record (EOF) without any I/O. // Browsers do this for every empty row they paint, so it must not cost a query. - SELF:_GoToPhantom(TRUE) + // ADS sets BOF as well, as for any record number that does not exist. + SELF:_GoToPhantom() + SELF:_SetBOF(TRUE) RETURN TRUE ENDIF SELF:_eofPending := FALSE @@ -811,7 +818,6 @@ partial class SQLRDD inherit Workarea SELF:RowNumber := rowIndex SELF:_SetEOF(FALSE) SELF:_SetBOF(FALSE) - SELF:_Found := TRUE return true endif rowIndex++ @@ -830,9 +836,16 @@ partial class SQLRDD inherit Workarea local found as logic if SELF:_tableMode == TableMode.Table .and. SELF:_recnoColumNo >= 0 .and. SELF:DataTable != null ; .and. !(SELF:_lazyGoto .and. !SELF:_bufferContiguous .and. SELF:RowCount <= 1) - found := SELF:_LoadSingleRecord(nRec) + var nFetched := SELF:_FetchSingleRecord(nRec) + found := nFetched == 1 if found SELF:_lazyGoto := TRUE + elseif nFetched == 0 + // The record does not exist (e.g. LastRec()+1, saved at EOF and restored): EOF without + // looking it up again in the order. ADS sets BOF as well (DBFCDX only for an empty file). + SELF:_GoToPhantom() + SELF:_SetBOF(TRUE) + RETURN FALSE else found := SELF:_GotoRecord(nRec) endif @@ -881,6 +894,9 @@ partial class SQLRDD inherit Workarea override method Skip(nToSkip as long) as logic LOCAL result := FALSE AS LOGIC SELF:_ForceOpen() + IF nToSkip != 0 + SELF:_Found := FALSE + ENDIF SELF:_Top := FALSE SELF:_Bottom := FALSE IF nToSkip != 0 @@ -889,6 +905,12 @@ partial class SQLRDD inherit Workarea ENDIF IF nToSkip == 0 result := SELF:GoCold() + ELSEIF nToSkip < 0 .AND. SELF:_EoF .AND. SELF:_BoF + // EOF and BOF at once (e.g. after GoTo() to a record that does not exist): ADS lands on + // the first record and keeps BOF, as for any backward skip at BOF. + SELF:GoCold() + result := SELF:GoTop() + SELF:BoF := TRUE ELSEIF SELF:_outsideOrder // Positioned (via GoTo()) on a record outside the current order (OrderKeyNo 0). // RowNumber/_bufferStart do not correspond to any real position in the order's diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg index 61f80f81d8..af0b8a4875 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg @@ -202,6 +202,8 @@ partial class SQLRDD aCurrentRow := SELF:CurrentRow:ItemArray endif var lTableCountValid := SELF:_tableReccountValid + // ADS keeps BOF as it was when the order changes at EOF + var lWasBof := SELF:_BoF if (orderInfo:Order is long var nOrder0 .and. nOrder0 == 0) .or. ; (orderInfo:Order is string var cOrder0 .and. String.IsNullOrEmpty(cOrder0)) // Order 0 (or an empty order name) means: switch back to the natural/physical @@ -221,7 +223,8 @@ partial class SQLRDD self:_tableReccountValid := lTableCountValid if lReset self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow) - else + self:_RestoreBofAtEof(lWasEof, lWasBof) + elseif !lWasEof self:GoTo(currentRecord) endif result := true @@ -242,7 +245,9 @@ partial class SQLRDD // be nothing to evaluate. if lReset self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow) - else + self:_RestoreBofAtEof(lWasEof, lWasBof) + elseif !lWasEof + // same order: at EOF stay there (GoTo(LastRec()+1) looked the record up again) self:GoTo(currentRecord) endif SELF:CurrentOrder:CalculateKeyLength() @@ -582,7 +587,7 @@ partial class SQLRDD IF SELF:DataTable == null SELF:GoTo(0) ELSE - SELF:_GoToPhantom(FALSE) + SELF:_GoToPhantom() ENDIF SELF:_Found := false SELF:_SetEOF(true) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 26cabf024f..fd3917320a 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -799,18 +799,16 @@ partial class SQLRDD return fields:ToArray() end method - /// Position on the phantom record (EOF) without reading anything. - /// FALSE when the caller sets BOF itself, which saves counting the - /// order for an empty buffer (e.g. a seek that found nothing). - private method _GoToPhantom(lSetBof as logic) as void + /// + /// Position on the phantom record (EOF) without reading anything. BOF is left to the caller: + /// GoTo() to a record that does not exist sets it (as ADS does), an order change at EOF and a + /// failed seek keep or clear it. Computing it here meant counting the order for an empty buffer. + /// + private method _GoToPhantom() as void SELF:_Found := FALSE SELF:_outsideOrder := FALSE SELF:RowNumber := SELF:RowCount + 1 SELF:_SetEOF(TRUE) - // BOF too only for an empty table (an empty buffer alone does not mean that) - if lSetBof - SELF:_SetBOF(SELF:RowCount == 0 .and. SELF:_OrderRowCount() == 0) - endif // When the buffer already ends at the last record, a backward Skip() from the phantom // row naturally lands on it; otherwise remember to go to the bottom first. SELF:_eofPending := !(SELF:_hasEOF .and. SELF:_bufferContiguous) @@ -1506,13 +1504,19 @@ partial class SQLRDD /// (Skip(), OrderKeyNo, ...). /// PRIVATE METHOD _LoadSingleRecord(nRec as DWORD) AS LOGIC + return SELF:_FetchSingleRecord(nRec) == 1 + + /// Like _LoadSingleRecord(): 1 = loaded, 0 = the record does not exist, -1 = error. + PRIVATE METHOD _FetchSingleRecord(nRec as DWORD) AS LONG try SELF:_lazyGoto := FALSE SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) SELF:_command:ClearParameters() var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) - if oTable == null .or. oTable:Rows:Count == 0 - return false + if oTable == null + return -1 + elseif oTable:Rows:Count == 0 + return 0 endif SELF:_ClearTable() SELF:DataTable := oTable @@ -1524,9 +1528,9 @@ partial class SQLRDD SELF:_bufferContiguous := false SELF:_SetEOF(FALSE) SELF:_SetBOF(FALSE) - return true + return 1 catch as Exception - return false + return -1 end try /// @@ -1548,7 +1552,8 @@ partial class SQLRDD // record. _GoToPhantom() marks the EOF as pending (the empty buffer does not end at // the last record), so Skip(1) stays at EOF and Skip(-1) goes to the bottom. if SELF:_SetBufferRows(null) .or. SELF:_OpenTable("1=0", "") - SELF:_GoToPhantom(TRUE) + // BOF stays as it was, see OrderListFocus() + SELF:_GoToPhantom() else SELF:GoTo(0) endif @@ -1562,6 +1567,13 @@ partial class SQLRDD endif return + /// After an order change at EOF keep BOF as it was before, as ADS does. + PRIVATE METHOD _RestoreBofAtEof(lWasEof as logic, lWasBof as logic) AS VOID + if lWasEof .and. SELF:_EoF + SELF:_SetBOF(lWasBof) + endif + return + /// /// Replace the buffer with an empty table (aRow NULL) or one row, built from the structure of /// the previous buffer without a query. FALSE when that structure is not known yet. From 83ba209d472ad611bf7438fb84128677c8cf386b Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Tue, 27 Oct 2026 11:36:48 +0100 Subject: [PATCH 03/13] SQLRDD: GoTo() returns TRUE at the phantom record GoTo() to a record that does not exist positions on the phantom record (EOF, and BOF as with ADS) and returned FALSE. The DBF RDD returns TRUE there, the positioning succeeded. The VO DbServer:GoTo() treats FALSE as an error: __DBSGoTo() builds an error object and BREAKs out, which in an ERP order copy (thousands of GoTo() back to a saved EOF position) took 7 % of the time. The eager path (a second GoTo() in a row) now also lands on the phantom record instead of staying on the previous one. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index aada7e06eb..f5b6f5cca8 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -843,15 +843,23 @@ partial class SQLRDD inherit Workarea elseif nFetched == 0 // The record does not exist (e.g. LastRec()+1, saved at EOF and restored): EOF without // looking it up again in the order. ADS sets BOF as well (DBFCDX only for an empty file). + // TRUE as in DBF: positioning on the phantom record succeeded. FALSE made the VO + // DbServer:GoTo() build and throw an error object every time. SELF:_GoToPhantom() SELF:_SetBOF(TRUE) - RETURN FALSE + RETURN TRUE else found := SELF:_GotoRecord(nRec) endif else found := SELF:_GotoRecord(nRec) endif + if !found .and. SELF:_tableMode == TableMode.Table + // does not exist, as above + SELF:_GoToPhantom() + SELF:_SetBOF(TRUE) + RETURN TRUE + endif SELF:_CheckEofBof() RETURN found end method From 8e971446ae347f4c081d65a3da1313add07bea00 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Tue, 27 Oct 2026 11:36:49 +0100 Subject: [PATCH 04/13] SQLRDD: less client work per statement Profiling an ERP order copy (1,600 mostly single-row statements) showed the client side costs per statement next to the server round trips: - Every query copied the structure of the buffer (_tableTemplate:Clone()) to load its rows into: 10 % of the time for tables with 80-180 columns. The replaced buffer, or a page whose rows were copied into the buffer, is now kept and the next result is loaded into it (SqlDbCommand.GetDataTable(cTable, oTemplate, oTarget)). - SqlDbTableCommandBuilder.ColumnList() built the quoted column list for every statement (4 %). It is now built once per table definition. Seek on a 178 column table: 1.24 -> 0.89 ms, SetOrder/GoTo/SetOrder 1.29 -> 0.93 ms. Results unchanged (navigation, EOF/BOF, sequences and seeks of 115 DTOS tags). Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/Classes/Command.prg | 6 ++- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 44 ++++++++++++++++--- .../Support/SqlDbTableCommandBuilder.prg | 13 ++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg index 712e83ab44..a4aa00e8ed 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg @@ -155,6 +155,8 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable /// /// The name of the DataTable (when the template cannot be used) /// An empty table with the columns the command returns, or NULL + /// An empty table with the structure of oTemplate to load the rows into instead + /// of a copy of oTemplate (copying the structure of a wide table is not free), or NULL. /// /// Building the structure from the reader's schema (DataTable.Load()) is the expensive part of /// a query that returns one or a few rows of a wide table: 1.5 ms against 0.5 ms for 178 @@ -162,7 +164,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable /// the structure of its buffer. When the reader's columns do not match the template, the /// rows are loaded as by GetDataTable(cTable). /// - method GetDataTable(cTable as string, oTemplate as DataTable) as DataTable + method GetDataTable(cTable as string, oTemplate as DataTable, oTarget := null as DataTable) as DataTable if oTemplate == null return SELF:GetDataTable(cTable) endif @@ -180,7 +182,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable next local oDataTable as DataTable if lMatch - oDataTable := oTemplate:Clone() + oDataTable := iif(oTarget != null, oTarget, oTemplate:Clone()) var aValues := object[]{reader:FieldCount} oDataTable:BeginLoadData() do while reader:Read() diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index fd3917320a..a0b3a63711 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -48,6 +48,9 @@ partial class SQLRDD private _bufferContiguous as logic /// Empty table with the buffer's columns, to load further query results into. private _tableTemplate as DataTable + /// An empty table with the buffer's structure (the last replaced buffer or a page already copied + /// into it) to load the next query result into, instead of copying _tableTemplate again. + private _spareTable as DataTable /// /// Rows to fetch with the next forward page (0 = PageSize): doubles with every full page read /// on in a row (up to BufferSize pages). After a seek it starts small, because code that @@ -155,7 +158,9 @@ partial class SQLRDD _table:Rows:Clear() return endif + var oOld := _table _table := value + SELF:_ReturnSpareTable(oOld) if _table is NULL SELF:_hasData := FALSE return @@ -633,7 +638,7 @@ partial class SQLRDD try SELF:_command:CommandText := _builder:BuildLastPageStatement(SELF:_SmallPageRows) SELF:_command:ClearParameters() - var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) + var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) if newTable == null return false endif @@ -647,6 +652,7 @@ partial class SQLRDD SELF:DataTable:Rows:InsertAt(newRow, 0) newRow:AcceptChanges() next + SELF:_ReturnSpareTable(newTable) SELF:_anchorWhere := "" SELF:_bufferContiguous := true SELF:_bufferStart := Math.Max(0, nTotalRows - SELF:RowCount) @@ -680,7 +686,7 @@ partial class SQLRDD try _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount, lReverse) _command:ClearParameters() - var oDataTable := _command:GetDataTable(self:Alias, SELF:_tableTemplate) + var oDataTable := _command:GetDataTable(self:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) return oDataTable catch as Exception return null @@ -1057,6 +1063,7 @@ partial class SQLRDD SELF:DataTable:Rows:Add(newRow) newRow:AcceptChanges() next + SELF:_ReturnSpareTable(newTable) // A short page always means EOF. But when the total record count is an exact // multiple of PageSize, the last page comes back FULL - "short page" never fires, // so also check whether this page's absolute record range already reaches the @@ -1108,6 +1115,7 @@ partial class SQLRDD SELF:DataTable:Rows:InsertAt(newRow, 0) newRow:AcceptChanges() next + SELF:_ReturnSpareTable(newTable) SELF:_bufferStart := nStart SELF:RowNumber += nRows SELF:_TrimBufferEnd() @@ -1149,6 +1157,7 @@ partial class SQLRDD SELF:DataTable:Rows:InsertAt(newRow, 0) newRow:AcceptChanges() next + SELF:_ReturnSpareTable(newTable) SELF:RowNumber += nRows SELF:_readAheadRows := 0 SELF:_readBackRows := iif(nRows < nCount, 0, Math.Min(nCount * 2, SELF:_MaxBufferRows)) @@ -1452,13 +1461,18 @@ partial class SQLRDD else SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) SELF:_command:ClearParameters() - var oRecTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) + var oRecTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) if oRecTable == null .or. oRecTable:Rows:Count == 0 + SELF:_ReturnSpareTable(oRecTable) return false endif oRow := oRecTable:Rows[0] endif var cFrom := SELF:_KeysetCondition(oRow, FALSE, TRUE, TRUE) + if oRow:Table != SELF:DataTable + // the record was read only for its key + SELF:_ReturnSpareTable(oRow:Table) + endif if cFrom == null return false endif @@ -1512,10 +1526,11 @@ partial class SQLRDD SELF:_lazyGoto := FALSE SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) SELF:_command:ClearParameters() - var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) + var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) if oTable == null return -1 elseif oTable:Rows:Count == 0 + SELF:_ReturnSpareTable(oTable) return 0 endif SELF:_ClearTable() @@ -1574,6 +1589,22 @@ partial class SQLRDD endif return + /// An empty table with the buffer's structure to load a query result into, or NULL. + PRIVATE METHOD _TakeSpareTable() AS DataTable + var oTable := SELF:_spareTable + SELF:_spareTable := null + return oTable + + /// Keep a table that is no longer used (a replaced buffer, a page whose rows were copied into + /// the buffer) for the next query, see _TakeSpareTable(). + PRIVATE METHOD _ReturnSpareTable(oTable as DataTable) AS VOID + if oTable != null .and. oTable != SELF:_table .and. SELF:_tableTemplate != null ; + .and. oTable:Columns:Count == SELF:_tableTemplate:Columns:Count + oTable:Rows:Clear() + SELF:_spareTable := oTable + endif + return + /// /// Replace the buffer with an empty table (aRow NULL) or one row, built from the structure of /// the previous buffer without a query. FALSE when that structure is not known yet. @@ -1582,7 +1613,10 @@ partial class SQLRDD if SELF:_tableTemplate == null return false endif - var oTable := SELF:_tableTemplate:Clone() + var oTable := SELF:_TakeSpareTable() + if oTable == null + oTable := SELF:_tableTemplate:Clone() + endif if aRow != null oTable:LoadDataRow(aRow, true) endif diff --git a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg index 538b5fc227..7790a61749 100644 --- a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg +++ b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg @@ -376,7 +376,20 @@ internal class SqlDbTableCommandBuilder return sb:ToString() + /// The quoted column list of the table, recno and deleted column last. + /// Built once: every statement of the work area uses it, and for a table with 178 columns + /// building it took a noticeable part of a seek on the client. method ColumnList() as string + if _columnList == null .or. _columnListCount != _oTable:Columns:Count + _columnList := SELF:_BuildColumnList() + _columnListCount := _oTable:Columns:Count + endif + return _columnList + + private _columnList as string + private _columnListCount as long + + private method _BuildColumnList() as string var sb := StringBuilder{} var List := Dictionary{StringComparer.OrdinalIgnoreCase} var newColumns := List{} From b5eff1aa4341d8535bb7d61a4cc82a8b9aff4dc2 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Tue, 27 Oct 2026 12:24:17 +0100 Subject: [PATCH 05/13] SQLRDD: short-lived result cache for repeated reads (ResultCacheMs) Code written for DBF/ADS repeats the same lookups again and again: in an ERP order copy 1,115 of 1,625 statements were exact repeats of an earlier read, with no write to the table in between (the same seek into a tax, item group or parameter table for every line). With a local index these are free, against a server each is a round trip plus loading the result. The new connection option ResultCacheMs (default 0 = off) lets the RDD answer a read statement it sent within that time from its result: - Only the RDD's own reads (SqlDbCommand.CacheTable, set for table work areas): a single SELECT without parameters, results of up to 20 rows. Larger results (pages of a sequential read) are not kept. - The entries of a table are dropped as soon as this connection writes to it (insert, update, delete of any work area) or locks one of its records, so a locked record is always read fresh. - All entries are dropped when a transaction is rolled back or when a statement that does not come from the RDD writes. - Changes of other users are seen at the latest after ResultCacheMs. Checked with two work areas and a second connection as another user: own writes and appends are seen at once, another user's change after the time or after RLock(). Results of the navigation, EOF/BOF, sequence and DTOS seek tests are identical with and without the cache; the navigation test sends 25-55 % fewer statements. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/Classes/Command.prg | 74 +++++++++++- .../XSharp.SQLRdd/Classes/Connection.prg | 33 ++++++ .../XSharp.SQLRdd/Classes/ResultCache.prg | 112 ++++++++++++++++++ src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 6 + .../XSharp.SQLRdd/XSharp.SQLRDD.xsproj | 1 + 5 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 src/Runtime/XSharp.SQLRdd/Classes/ResultCache.prg diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg index a4aa00e8ed..8318f575a2 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg @@ -33,9 +33,45 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable property Provider as ISqlDbProvider get iif(Connection == null, null, Connection:Provider) /// The text of the Command object. property CommandText as string get iif(DbCommand == null, "", DbCommand:CommandText) set DbCommand:CommandText := value + /// The table the RDD reads with this command: its read statements may be answered from the + /// connection's result cache (see SqlDbConnection.ResultCacheMs). NULL = never. + internal property CacheTable as string auto #endregion + /// The statement as cache key, or NULL when its result may not be reused: no cache, not an RDD + /// read (CacheTable), parameters, or not a single SELECT. + private method _CacheKey() as string + if String.IsNullOrEmpty(SELF:CacheTable) .or. SELF:Connection:ResultCache == null .or. SELF:Parameters?:Count > 0 + return null + endif + var cText := SELF:CommandText + if cText == null .or. cText:IndexOf(';') >= 0 .or. !cText:TrimStart():StartsWith("select", StringComparison.OrdinalIgnoreCase) + return null + endif + return cText + end method + + /// Before a statement that is not a SELECT runs: drop the cached results it may change, those of + /// CacheTable for a write of the RDD, all of them for a statement of somebody else. + private method _InvalidateCache() as void + var oCache := SELF:Connection:ResultCache + if oCache == null + return + endif + var cText := SELF:CommandText + if cText == null .or. cText:TrimStart():StartsWith("select", StringComparison.OrdinalIgnoreCase) .and. cText:IndexOf(';') < 0 + return + endif + if !String.IsNullOrEmpty(SELF:CacheTable) + oCache:Invalidate(SELF:CacheTable) + elseif cText:IndexOf(SqlDbConnection.LockTableName, StringComparison.OrdinalIgnoreCase) < 0 + // the lock table refresh runs on a timer and changes no data + oCache:Clear() + endif + return + end method + private withEvents as LOGIC #region static properties and methods @@ -169,6 +205,18 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable return SELF:GetDataTable(cTable) endif try + local oCached as SqlDbResultCacheEntry + var cKey := SELF:_CacheKey() + if cKey != null .and. SELF:Connection:ResultCache:TryGet(cKey, out oCached) .and. !oCached:IsScalar ; + .and. oCached:Columns == oTemplate:Columns:Count + var oResult := iif(oTarget != null, oTarget, oTemplate:Clone()) + oResult:BeginLoadData() + foreach var aRow in oCached:Rows + oResult:LoadDataRow(aRow, true) + next + oResult:EndLoadData() + return oResult + endif using var reader := SELF:ExecuteReader(cTable) if reader == null return null @@ -184,12 +232,24 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable if lMatch oDataTable := iif(oTarget != null, oTarget, oTemplate:Clone()) var aValues := object[]{reader:FieldCount} + var aRows := iif(cKey != null, List{}, null) oDataTable:BeginLoadData() do while reader:Read() reader:GetValues(aValues) oDataTable:LoadDataRow(aValues, true) + if aRows != null + if aRows:Count < SqlDbResultCache.MaxRows + aRows:Add((object[]) aValues:Clone()) + else + aRows := null + cKey := null + endif + endif enddo oDataTable:EndLoadData() + if aRows != null + SELF:Connection:ResultCache:Put(cKey, SqlDbResultCacheEntry{}{Table := SELF:CacheTable, Rows := aRows, Columns := reader:FieldCount}) + endif else oDataTable := DataTable{iif(String.IsNullOrEmpty(cTable), SELF:Name, cTable)} var oDataSet := DataSet{} @@ -214,6 +274,12 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable method ExecuteScalar(cTable := "" as string) as object try self:Connection:ForceOpen() + local oCached as SqlDbResultCacheEntry + var cKey := SELF:_CacheKey() + if cKey != null .and. SELF:Connection:ResultCache:TryGet(cKey, out oCached) .and. oCached:IsScalar + return oCached:Scalar + endif + SELF:_InvalidateCache() if String.IsNullOrEmpty(cTable) cTable := SELF:Name endif @@ -224,7 +290,11 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable if ! SELF:_TryBindParameters() return null endif - return self:DbCommand:ExecuteScalar() + var oResult := self:DbCommand:ExecuteScalar() + if cKey != null + SELF:Connection:ResultCache:Put(cKey, SqlDbResultCacheEntry{}{Table := SELF:CacheTable, Scalar := oResult, IsScalar := TRUE}) + endif + return oResult catch e as Exception Connection:LastException := e return null @@ -241,6 +311,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable method ExecuteReader(cTable := "" as string) as DbDataReader try self:Connection:ForceOpen() + SELF:_InvalidateCache() if String.IsNullOrEmpty(cTable) cTable := SELF:Name endif @@ -268,6 +339,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable method ExecuteNonQuery(cTable := "" as string) as LOGIC try self:Connection:ForceOpen() + SELF:_InvalidateCache() if String.IsNullOrEmpty(cTable) cTable := SELF:Name endif diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg index 1ce955f251..89ba34223f 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg @@ -111,6 +111,33 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable /// property UseLocks as logic auto + /// + /// How long (milliseconds) the RDD may reuse the result of an identical read statement (default 0 = never). + /// + /// + /// Code written for DBF/ADS repeats the same lookups again and again, a round trip each against a + /// server. With a value > 0 the RDD answers a read statement it sent within that time from its + /// result. A table's results are dropped as soon as this connection writes to it or locks one of its + /// records, and all of them when a transaction is rolled back or a statement not sent by the RDD + /// writes. Changes of other users are seen at the latest after that time. + /// + property ResultCacheMs as long auto + + private _resultCache as SqlDbResultCache + /// The cache for ResultCacheMs, NULL when it is off. + internal property ResultCache as SqlDbResultCache + get + if SELF:ResultCacheMs <= 0 + return null + endif + if _resultCache == null + _resultCache := SqlDbResultCache{SELF:ResultCacheMs} + endif + _resultCache:MaxAge := SELF:ResultCacheMs + return _resultCache + end get + end property + /// Identifier Case as returned by the Ado.Net provider in the DataSourceInformation metadata collection. property IdentifierCase as System.Data.Common.IdentifierCase auto get private set /// Quoted Identifier Case as returned by the Ado.Net provider in the DataSourceInformation metadata collection. @@ -222,6 +249,10 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable self:UseNulls := isTrue case "uselocks" self:UseLocks := isTrue + case "resultcachems" + if Int32.TryParse(strValue, out var cacheMs) + self:ResultCacheMs := cacheMs + endif otherwise builder:Add(key, strValue) end switch @@ -461,6 +492,8 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable /// method RollBackTrans as logic if self:DbTransaction != null + // results read inside the transaction may show changes that are now undone + self:_resultCache?:Clear() try self:DbTransaction:Rollback() self:DbTransaction := null diff --git a/src/Runtime/XSharp.SQLRdd/Classes/ResultCache.prg b/src/Runtime/XSharp.SQLRdd/Classes/ResultCache.prg new file mode 100644 index 0000000000..ffd048da66 --- /dev/null +++ b/src/Runtime/XSharp.SQLRdd/Classes/ResultCache.prg @@ -0,0 +1,112 @@ +// +// Copyright (c) XSharp B.V. All Rights Reserved. +// Licensed under the Apache License, Version 2.0. +// See License.txt in the project root for license information. +// + +using System +using System.Collections.Generic +using System.Diagnostics + +begin namespace XSharp.RDD.SqlRDD + +/// +/// Results of identical read statements of the RDD, reused for a short time +/// (see SqlDbConnection.ResultCacheMs). +/// +/// +/// Code written for DBF/ADS repeats the same lookups over and over (the same seek into a tax or +/// parameter table for every line of a document): free with a local index, a round trip each with +/// a server. The entries of a table are dropped as soon as this connection writes to the table or +/// locks a record in it; changes of other users are seen at the latest when an entry expires. +/// +internal class SqlDbResultCache + private _entries as Dictionary + private _lock as object + private const MaxEntries := 1000 as long + /// Larger results (pages of a sequential read) are not kept: they are hardly repeated, and + /// copying them cost more than it saved. + internal const MaxRows := 20 as long + + /// Maximum age of an entry in milliseconds. + property MaxAge as long auto + + constructor(nMaxAge as long) + _entries := Dictionary{StringComparer.Ordinal} + _lock := object{} + MaxAge := nMaxAge + return + + method TryGet(cStatement as string, oEntry out SqlDbResultCacheEntry) as logic + begin lock _lock + if _entries:TryGetValue(cStatement, out oEntry) + if (Stopwatch.GetTimestamp() - oEntry:Created) * 1000 / Stopwatch.Frequency <= MaxAge + return true + endif + _entries:Remove(cStatement) + endif + end lock + oEntry := null + return false + + method Put(cStatement as string, oEntry as SqlDbResultCacheEntry) as void + oEntry:Created := Stopwatch.GetTimestamp() + begin lock _lock + if _entries:Count >= MaxEntries + SELF:_RemoveExpired() + if _entries:Count >= MaxEntries + _entries:Clear() + endif + endif + _entries[cStatement] := oEntry + end lock + return + + private method _RemoveExpired() as void + var nNow := Stopwatch.GetTimestamp() + var aKeys := List{} + foreach var oPair in _entries + if (nNow - oPair:Value:Created) * 1000 / Stopwatch.Frequency > MaxAge + aKeys:Add(oPair:Key) + endif + next + foreach var cKey in aKeys + _entries:Remove(cKey) + next + return + + /// Drop the entries of a table. + method Invalidate(cTable as string) as void + begin lock _lock + if _entries:Count > 0 + var aKeys := List{} + foreach var oPair in _entries + if String.Equals(oPair:Value:Table, cTable, StringComparison.OrdinalIgnoreCase) + aKeys:Add(oPair:Key) + endif + next + foreach var cKey in aKeys + _entries:Remove(cKey) + next + endif + end lock + return + + method Clear() as void + begin lock _lock + _entries:Clear() + end lock + return +end class + +/// The result of one statement: rows (object arrays) or a scalar. +internal class SqlDbResultCacheEntry + property Table as string auto + property Created as int64 auto + property Rows as List auto + property Columns as long auto + property Scalar as object auto + property IsScalar as logic auto +end class + +end namespace diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index f5b6f5cca8..f5ef858ba6 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -150,6 +150,8 @@ partial class SQLRDD inherit Workarea if !self:_oTd:HasRecnoColumn throw Exception{"RecnoColumn is required for table " + info:FileName} endif + // its read statements may be answered from the result cache (SqlDbConnection.ResultCacheMs) + _command:CacheTable := self:_oTd:RealName endif local aUpdatableColumns := null as HashSet if _oTd:HasUpdatableColumns @@ -1036,6 +1038,10 @@ partial class SQLRDD inherit Workarea lockInfo:Result := false return false endif + // Read fresh after a lock: the record may be written next. + if self:_oTd != null + self:Connection:ResultCache?:Invalidate(self:_oTd:RealName) + endif if !self:Connection:UseLocks // Locking switched off for this connection (see SqlDbConnection.UseLocks). lockInfo:Result := true diff --git a/src/Runtime/XSharp.SQLRdd/XSharp.SQLRDD.xsproj b/src/Runtime/XSharp.SQLRdd/XSharp.SQLRDD.xsproj index 06f6838ed1..63a9fa9473 100644 --- a/src/Runtime/XSharp.SQLRdd/XSharp.SQLRDD.xsproj +++ b/src/Runtime/XSharp.SQLRdd/XSharp.SQLRDD.xsproj @@ -57,6 +57,7 @@ + From 0f5012b8f64f5b3e008920868664112c9ac89a88 Mon Sep 17 00:00:00 2001 From: Robert van der Hulst Date: Tue, 27 Oct 2026 17:02:38 +0100 Subject: [PATCH 06/13] Update condition to include backslash check Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index a0b3a63711..0980c8bbab 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -994,7 +994,7 @@ partial class SQLRDD switch oRow[nCol] case cValue as string foreach var c in cValue - if c > (char) 127 + if c > (char) 127 .or. c == (char) 92 return null endif next From 3899c46a957dece8f9e078cf80876b957c90d180 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:03:23 +0100 Subject: [PATCH 07/13] SQLRDD: write pending changes before Skip() looks up the position After GoTo() loaded only the record, Skip() looked up its position in the order first and wrote the pending changes afterwards. Loading the window replaced the buffer, so GoTo(n), FieldPut(), Skip() lost the change without an error. The changes are now written before the window is loaded, and Skip() returns FALSE when that write fails. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 4 +++- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index f5ef858ba6..3ff0a9c2d2 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -911,7 +911,9 @@ partial class SQLRDD inherit Workarea SELF:_Bottom := FALSE IF nToSkip != 0 // e.g. right after SetOrder(): the record's position in the order is still unknown - SELF:_EnsurePositioned() + IF !SELF:_EnsurePositioned() + RETURN FALSE + ENDIF ENDIF IF nToSkip == 0 result := SELF:GoCold() diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 0980c8bbab..8ee8bb8593 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -1640,13 +1640,19 @@ partial class SQLRDD /// When the buffer only holds records whose position in the current order is unknown /// (see _LoadSingleRecord()), look that position up for the current record now. /// Sets _outsideOrder when the record is not part of the order at all. + /// Returns FALSE when pending changes could not be written. /// - PRIVATE METHOD _EnsurePositioned() AS VOID + PRIVATE METHOD _EnsurePositioned() AS LOGIC if SELF:_tableMode == TableMode.Table .and. !SELF:_bufferContiguous .and. !SELF:_outsideOrder ; .and. !SELF:_EoF .and. SELF:RowCount > 0 .and. SELF:RowNumber >= 1 .and. SELF:RowNumber <= SELF:RowCount + // Loading the window replaces the buffer: write the changes of the record first, or + // GoTo(n), FieldPut(), Skip() would lose them. + if !SELF:GoCold() + return false + endif SELF:_GotoRecord(Convert.ToUInt32(SELF:CurrentRow[self:_recnoColumNo])) endif - return + return true PRIVATE METHOD _GotoRow(nRow as LONG) AS LOGIC SELF:_Found := FALSE From ac58e734791b356ae76eca8da89475ddfd1d2120 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:04:55 +0100 Subject: [PATCH 08/13] SQLRDD: check the outcome of the insert in Append() The command methods report a failure through their result and LastException instead of throwing. A failed insert was only noticed when assigning the missing identity happened to throw, and for providers without an identity query, the failed insert and the commit were not checked at all. Each step is now checked, and the original error is reported. Co-Authored-By: Claude Opus 5.5 --- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 8ee8bb8593..b7e3735dd2 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -411,7 +411,7 @@ partial class SQLRDD hasGetIdentity := true endif try - _command:Connection:BeginTrans() + var lOwnTrans := _command:Connection:BeginTrans() _command:BindParameters() local lInsertWithGetIdentity := false as LOGIC if hasGetIdentity .and. ! String.IsNullOrEmpty(Provider:GetIdentity) @@ -420,16 +420,29 @@ partial class SQLRDD lInsertWithGetIdentity := TRUE endif _command.CommandText := Connection:RaiseStringEvent(_command, SqlRDDEventReason.CommandText, _cTable, sb:ToString()) + // The command methods report a failure through their result and LastException, they + // do not throw. + local lOk as logic + _command:Connection:LastException := null if lInsertWithGetIdentity var result := _command:ExecuteScalar() - row[_recnoColumNo] := result - elseif hasGetIdentity - _command:ExecuteScalar() - row[_recnoColumNo] := _builder:GetMaxRecno() + lOk := result != null .and. result != DBNull.Value + if lOk + row[_recnoColumNo] := result + endif else - _command:ExecuteNonQuery() + lOk := _command:ExecuteNonQuery() + if lOk .and. hasGetIdentity + row[_recnoColumNo] := _builder:GetMaxRecno() + endif + endif + if lOk + lOk := _command:Connection:CommitTrans() .or. !lOwnTrans + endif + if !lOk + var oError := _command:Connection:LastException + throw iif(oError != null, oError, Exception{"The insert returned no record number"}) endif - _command:Connection:CommitTrans() catch e as Exception // Reporting success here made a failed insert look like a new record whose changes // then went nowhere. From 459b5cc9d6ae3997262eb311a4b38ff0eacd9aca Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:05:02 +0100 Subject: [PATCH 09/13] SQLRDD: a failed Append() leaves the EOF state as it was Append() cleared the pending EOF and marked the buffer as not contiguous before the insert. When the insert failed, the work area stayed where it was, but with these flags changed: at a pending EOF, the next Skip(1) read on behind the buffer. They are now changed only after a successful insert. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 3ff0a9c2d2..076c22c140 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -281,14 +281,11 @@ partial class SQLRDD inherit Workarea endif var lResult := SELF:GoCold() if lResult - self:_eofPending := false // With a recno column the insert below returns the new record number; asking the server // for MAX(recno) first only cost a round trip per Append(). var key := iif(self:_recnoColumNo != -1, 0U, (dword) self:_builder:GetNextKey()) var row := self:DataTable:NewRow() self:DataTable:Rows:Add(row) - // The new row does not belong at the end of the buffered slice of the order. - self:_bufferContiguous := false if _emptyValues == null self:_GetEmptyValues() endif @@ -301,10 +298,13 @@ partial class SQLRDD inherit Workarea endif next if !self:_ExecuteInsertStatement(row) - // the error is reported, stay where we were + // the error is reported, stay where we were (buffer and EOF state unchanged) self:DataTable:Rows:Remove(row) return false endif + self:_eofPending := false + // The new row does not belong at the end of the buffered slice of the order. + self:_bufferContiguous := false row:AcceptChanges() if self:_recnoColumNo != -1 key := Convert.ToUInt32(row[self:_recnoColumNo]) From cb8c99b1c1f64788e3fb7a16a17a29854564e065 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:05:03 +0100 Subject: [PATCH 10/13] SQLRDD: GoTo() reports a failed query instead of going to EOF GoTo() went to the phantom record whenever the record was not found. A failed query (timeout, missing permission) is no proof that the record does not exist, but it ended at EOF and GoTo() returned TRUE. When the lookup registered a new error, GoTo() now reports it and returns FALSE. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 076c22c140..e5e381a7c1 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -836,6 +836,7 @@ partial class SQLRDD inherit Workarea // rows instead of one query per row. (A single record left by an order change does not // count - SetOrder() followed by GoTo(saved record) is the usual restore pattern.) local found as logic + var oLastError := SELF:Connection:LastException if SELF:_tableMode == TableMode.Table .and. SELF:_recnoColumNo >= 0 .and. SELF:DataTable != null ; .and. !(SELF:_lazyGoto .and. !SELF:_bufferContiguous .and. SELF:RowCount <= 1) var nFetched := SELF:_FetchSingleRecord(nRec) @@ -857,6 +858,12 @@ partial class SQLRDD inherit Workarea found := SELF:_GotoRecord(nRec) endif if !found .and. SELF:_tableMode == TableMode.Table + // A failed query does not mean that the record does not exist (timeout, no permission ...) + var oError := SELF:Connection:LastException + if oError != null .and. oError != oLastError + SELF:_dbfError(ERDD.READ, XSharp.Gencode.EG_READ, "SqlRDD:GoTo", oError:Message, FALSE) + RETURN FALSE + endif // does not exist, as above SELF:_GoToPhantom() SELF:_SetBOF(TRUE) From 58fa6291087d14661f7bbed6ab56e3604c6d742a Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:05:29 +0100 Subject: [PATCH 11/13] SQLRDD: drop the cached results when ResultCacheMs is switched off While the cache is off, nothing is invalidated. Results kept from before could come back when it was switched on again within their lifetime, even after this connection had changed the rows. Switching it off now drops them. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/Classes/Connection.prg | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg index 89ba34223f..9dc87680fd 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg @@ -121,8 +121,21 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable /// records, and all of them when a transaction is rolled back or a statement not sent by the RDD /// writes. Changes of other users are seen at the latest after that time. /// - property ResultCacheMs as long auto + property ResultCacheMs as long + get + return _resultCacheMs + end get + set + _resultCacheMs := value + // While it is off nothing is invalidated: results from before must not come back + // when it is switched on again. + if value <= 0 + _resultCache := null + endif + end set + end property + private _resultCacheMs as long private _resultCache as SqlDbResultCache /// The cache for ResultCacheMs, NULL when it is off. internal property ResultCache as SqlDbResultCache From 0e5ea36a077a2c36ff27a7d252b95696902bace0 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:06:22 +0100 Subject: [PATCH 12/13] SQLRDD: result cache keyed by the statement that is sent, writes in batches - The cache key was taken before the CommandText callback rewrote the statement, and a cached result skipped the callback. Two work areas whose statements only differ after the callback could share results. The callback now runs first, exactly once, also for a cached result. - A batch needs no semicolon between its statements, so a SELECT followed by an UPDATE looked like a read: it was not invalidating the cache. A statement with a word that writes (INSERT, UPDATE, DELETE, MERGE, INTO, EXEC, DDL) now counts as a write. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/Classes/Command.prg | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg index 8318f575a2..5aad8a2cce 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg @@ -39,14 +39,32 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable #endregion + private static _writeWords := System.Text.RegularExpressions.Regex{ ; + "\b(insert|update|delete|merge|into|exec|execute|create|alter|drop|truncate)\b", ; + System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.Compiled} as System.Text.RegularExpressions.Regex + + /// TRUE for a single SELECT that writes nothing. + /// + /// A batch does not need a semicolon between its statements, so for a statement that does not come + /// from the RDD (no CacheTable) a word that writes counts as a write (also when the word is part of a + /// literal: then the cache is just dropped once too often). The RDD's own reads are SELECTs it builds, + /// and searching them costs more than the statement itself: a page of a table with 150 columns is a + /// few KB of text. + /// + private method _IsReadOnlySelect(cText as string) as logic + return cText:TrimStart():StartsWith("select", StringComparison.OrdinalIgnoreCase) .and. cText:IndexOf(';') < 0 ; + .and. (!String.IsNullOrEmpty(SELF:CacheTable) .or. !_writeWords:IsMatch(cText)) + end method + /// The statement as cache key, or NULL when its result may not be reused: no cache, not an RDD - /// read (CacheTable), parameters, or not a single SELECT. + /// read (CacheTable), parameters, or not a single SELECT. Call it after CallBack(), the key is the + /// statement that is sent. private method _CacheKey() as string if String.IsNullOrEmpty(SELF:CacheTable) .or. SELF:Connection:ResultCache == null .or. SELF:Parameters?:Count > 0 return null endif var cText := SELF:CommandText - if cText == null .or. cText:IndexOf(';') >= 0 .or. !cText:TrimStart():StartsWith("select", StringComparison.OrdinalIgnoreCase) + if cText == null .or. !SELF:_IsReadOnlySelect(cText) return null endif return cText @@ -60,7 +78,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable return endif var cText := SELF:CommandText - if cText == null .or. cText:TrimStart():StartsWith("select", StringComparison.OrdinalIgnoreCase) .and. cText:IndexOf(';') < 0 + if cText == null .or. SELF:_IsReadOnlySelect(cText) return endif if !String.IsNullOrEmpty(SELF:CacheTable) @@ -205,6 +223,11 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable return SELF:GetDataTable(cTable) endif try + if String.IsNullOrEmpty(cTable) + cTable := SELF:Name + endif + // the cache key is the statement as the callback leaves it, and a cached result passes it as well + SELF:CallBack(cTable) local oCached as SqlDbResultCacheEntry var cKey := SELF:_CacheKey() if cKey != null .and. SELF:Connection:ResultCache:TryGet(cKey, out oCached) .and. !oCached:IsScalar ; @@ -217,7 +240,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable oResult:EndLoadData() return oResult endif - using var reader := SELF:ExecuteReader(cTable) + using var reader := SELF:_ExecuteReader(cTable, FALSE) if reader == null return null endif @@ -274,16 +297,16 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable method ExecuteScalar(cTable := "" as string) as object try self:Connection:ForceOpen() + if String.IsNullOrEmpty(cTable) + cTable := SELF:Name + endif + SELF:CallBack(cTable) local oCached as SqlDbResultCacheEntry var cKey := SELF:_CacheKey() if cKey != null .and. SELF:Connection:ResultCache:TryGet(cKey, out oCached) .and. oCached:IsScalar return oCached:Scalar endif SELF:_InvalidateCache() - if String.IsNullOrEmpty(cTable) - cTable := SELF:Name - endif - SELF:CallBack(cTable) if self:Connection:DbTransaction != null self:DbTransaction := self:Connection:DbTransaction endif @@ -309,13 +332,19 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable /// When an error occurs then the error is registered in the LastException property of the Connection /// method ExecuteReader(cTable := "" as string) as DbDataReader + return SELF:_ExecuteReader(cTable, TRUE) + end method + + private method _ExecuteReader(cTable as string, lCallBack as logic) as DbDataReader try self:Connection:ForceOpen() - SELF:_InvalidateCache() if String.IsNullOrEmpty(cTable) cTable := SELF:Name endif - SELF:CallBack(cTable) + if lCallBack + SELF:CallBack(cTable) + endif + SELF:_InvalidateCache() if self:Connection:DbTransaction != null self:DbTransaction := self:Connection:DbTransaction endif From 7be52398feccbcc511a4b70bf71332ca08f784a2 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Wed, 28 Oct 2026 08:07:32 +0100 Subject: [PATCH 13/13] SQLRDD: read the record again after a new record lock Code written for DBF/ADS locks a record, reads it and writes it back. The work area kept the values it had read before the lock (from its buffer or from the result cache), so a change another user committed in the meantime could be overwritten. Like ADS, a new lock of the record the work area stands on now reads it again, unless the work area has pending changes. With UseLocks off nothing is locked and nothing is read again. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 5 ++- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 42 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index e5e381a7c1..fc2d5a24af 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -1052,7 +1052,8 @@ partial class SQLRDD inherit Workarea self:Connection:ResultCache?:Invalidate(self:_oTd:RealName) endif if !self:Connection:UseLocks - // Locking switched off for this connection (see SqlDbConnection.UseLocks). + // Locking switched off for this connection (see SqlDbConnection.UseLocks). Nothing is + // locked, so the record is not read again either (see _RefreshLockedRow()). lockInfo:Result := true return true endif @@ -1078,6 +1079,7 @@ partial class SQLRDD inherit Workarea if myLock SELF:_ownLocks:Add(nLockRecNo) + SELF:_RefreshLockedRow(nLockRecNo) lockInfo:Result := true return true endif @@ -1115,6 +1117,7 @@ partial class SQLRDD inherit Workarea end try SELF:_ownLocks:Add(nLockRecNo) + SELF:_RefreshLockedRow(nLockRecNo) lockInfo:Result := true return true end method diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index b7e3735dd2..a08a06534d 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -1608,6 +1608,48 @@ partial class SQLRDD SELF:_spareTable := null return oTable + /// + /// After a new record lock: read the locked record again when the work area stands on it and has + /// no pending changes, as ADS does. + /// + /// + /// Code written for DBF/ADS locks a record, reads it and writes it back. A value read before the + /// lock (from the buffer or the result cache) could overwrite a change another user committed in + /// the meantime. Only the values of the row are replaced, the buffer and the position stay. + /// + PRIVATE METHOD _RefreshLockedRow(nRecNo as int) AS VOID + var row := SELF:CurrentRow + if nRecNo <= 0 .or. SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo < 0 .or. SELF:_EoF ; + .or. row == null .or. row == SELF:_phantomRow .or. row:RowState != DataRowState.Unchanged .or. SELF:_updatedRecNos:Count > 0 + return + endif + if Convert.ToInt32(row[SELF:_recnoColumNo]) != nRecNo + return + endif + try + SELF:_command:CommandText := _builder:BuildDirectRecnoStatement((DWORD) nRecNo) + SELF:_command:ClearParameters() + var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) + if oTable != null + if oTable:Rows:Count == 1 + var oNew := oTable:Rows[0] + // column by column: the recno column is read only (ItemArray would stop there + // and leave the row in edit mode) + foreach oColumn as DataColumn in row:Table:Columns + if !oColumn:ReadOnly + row[oColumn] := oNew[oColumn:Ordinal] + endif + next + row:AcceptChanges() + endif + SELF:_ReturnSpareTable(oTable) + endif + catch as Exception + // keep the values read before + row:RejectChanges() + end try + return + /// Keep a table that is no longer used (a replaced buffer, a page whose rows were copied into /// the buffer) for the next query, see _TakeSpareTable(). PRIVATE METHOD _ReturnSpareTable(oTable as DataTable) AS VOID