From e1474893332b2d7ce74446f98427894513399150 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 11:22:49 +0100 Subject: [PATCH 1/7] SQLRDD: load query results into the buffer's structure Every query built a new DataTable from the reader's schema (DataTable.Load()). For a query that returns one or a few rows of a wide table that is most of the client side cost: measured 1.5 ms per query for 178 columns against 0.5 ms when the rows are loaded into an existing structure. Interactive work (seeks, relations, GoTo()) is mostly such queries - an order entry form of our ERP runs about 140 of them per paging step. The work area keeps an empty clone of its first buffer (all its queries use the same column list) and SqlDbCommand.GetDataTable(cTable, oTemplate) loads the rows into a copy of it with GetValues()/LoadDataRow(). If the reader's columns do not match the template, the rows are loaded as before. Seek on a 178 column table: 4.2 -> 3.3 ms, SetOrder/GoTo/SetOrder: 11.1 -> 8.3 ms per round (including the server round trips). Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/Classes/Command.prg | 52 +++++++++++++++++++ .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 13 +++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg index 0fa68e1e5b..712e83ab44 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg @@ -150,6 +150,58 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable end try end method + /// + /// Execute the command and return the rows in a copy of oTemplate's structure. + /// + /// The name of the DataTable (when the template cannot be used) + /// An empty table with the columns the command returns, or NULL + /// + /// Building the structure from the reader's schema (DataTable.Load()) is the expensive part of + /// a query that returns one or a few rows of a wide table: 1.5 ms against 0.5 ms for 178 + /// columns. The RDD runs every query of a work area with the same column list, so it passes + /// the structure of its buffer. When the reader's columns do not match the template, the + /// rows are loaded as by GetDataTable(cTable). + /// + method GetDataTable(cTable as string, oTemplate as DataTable) as DataTable + if oTemplate == null + return SELF:GetDataTable(cTable) + endif + try + using var reader := SELF:ExecuteReader(cTable) + if reader == null + return null + endif + var lMatch := reader:FieldCount == oTemplate:Columns:Count + for var i := 0 to reader:FieldCount-1 + if !lMatch + exit + endif + lMatch := String.Equals(reader:GetName(i), oTemplate:Columns[i]:ColumnName, StringComparison.OrdinalIgnoreCase) + next + local oDataTable as DataTable + if lMatch + oDataTable := oTemplate:Clone() + var aValues := object[]{reader:FieldCount} + oDataTable:BeginLoadData() + do while reader:Read() + reader:GetValues(aValues) + oDataTable:LoadDataRow(aValues, true) + enddo + oDataTable:EndLoadData() + else + oDataTable := DataTable{iif(String.IsNullOrEmpty(cTable), SELF:Name, cTable)} + var oDataSet := DataSet{} + oDataSet:EnforceConstraints := false + oDataSet:Tables:Add(oDataTable) + oDataTable:Load(reader) + endif + return oDataTable + catch e as Exception + Connection:LastException := e + return null + end try + end method + /// /// Execute a scalar command /// diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 61d6302fd1..92d398ad81 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -46,6 +46,8 @@ partial class SQLRDD /// the buffer edges then has to re-locate the current record first. /// private _bufferContiguous as logic + /// Empty table with the buffer's columns, to load further query results into. + private _tableTemplate as DataTable /// /// Rows to fetch with the next forward page (0 = PageSize): doubles with every full page read /// on in a row (up to BufferSize pages). After a seek it starts small, because code that @@ -177,6 +179,11 @@ partial class SQLRDD endif dbColumn:Flags := DBFFieldFlags.None next + if SELF:_tableTemplate == null + // the structure for all further queries of this work area (same column list), + // see SqlDbCommand.GetDataTable(cTable, oTemplate) + SELF:_tableTemplate := _table:Clone() + endif SELF:RowNumber := 1 SELF:_CheckEofBof() end set @@ -615,7 +622,7 @@ partial class SQLRDD try SELF:_command:CommandText := _builder:BuildLastPageStatement() SELF:_command:ClearParameters() - var newTable := SELF:_command:GetDataTable(SELF:Alias) + var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) if newTable == null return false endif @@ -660,7 +667,7 @@ partial class SQLRDD try _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount) _command:ClearParameters() - var oDataTable := _command:GetDataTable(self:Alias) + var oDataTable := _command:GetDataTable(self:Alias, SELF:_tableTemplate) return oDataTable catch as Exception return null @@ -1266,7 +1273,7 @@ partial class SQLRDD try SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) SELF:_command:ClearParameters() - var oTable := SELF:_command:GetDataTable(SELF:Alias) + var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) if oTable == null .or. oTable:Rows:Count == 0 return false endif From 758712232a44cd7b4c7342eb7178c7b2b9c774e1 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 10:59:19 +0100 Subject: [PATCH 2/7] SQLRDD: GoTo() loads only the record, its position is looked up when needed GoTo() to a record that is not buffered looked up its position in the current order right away: a ROW_NUMBER() over the order plus a whole page around the record. Code that saves the record number, seeks elsewhere and goes back, or that follows relations, mostly just reads the record's fields afterwards; in an ERP order entry form paging from one order to the next did this 4-5 times per step on the order header table alone (each time ROW_NUMBER() and two pages of a 150 column table). GoTo() now loads just the record by its recno, the way an order change already does (_LoadSingleRecord()), and the position is looked up by _EnsurePositioned()/_AbsolutePosition() when Skip(), OrderKeyNo etc. need it. When the buffer already holds such a single record, GoTo() positions fully as before, so a browse that calls GoTo() for each row it paints still gets one page with the neighbouring rows instead of one query per row. DBI_GETLOCKARRAY/DBI_LOCKCOUNT no longer query the lock table when UseLocks is off: no lock entries are written then, and code that checks the lock list after every movement (RLockList, IsLocked()) paid a round trip each time. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 7257f8d665..e30dd42d50 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -616,6 +616,11 @@ partial class SQLRDD inherit Workarea if Connection?:Provider is null .or. !self:Connection:IsOpen .or. _oTd == null return recNos:ToArray() endif + if !self:Connection:UseLocks + // No lock table entries are ever written without UseLocks. Code that asks for the + // lock list after every movement (RLockList/IsLocked()) paid a query each time. + return recNos:ToArray() + endif try var sb := StringBuilder{} sb:AppendLine("select recno from " + SqlDbConnection.LockTableName) @@ -804,7 +809,24 @@ partial class SQLRDD inherit Workarea next endif - var found := SELF:_GotoRecord(nRec) + // Code that saves the record number, seeks elsewhere and goes back (or follows a + // relation) mostly just reads the record's fields afterwards. Its position in the order + // is only needed when something moves from it (Skip(), OrderKeyNo ...), see + // _EnsurePositioned(): load just the record by its recno. Looking the position up right + // away cost a ROW_NUMBER() over the order plus a whole page around the record. + // When the buffer already is such a single record (the previous GoTo() was lazy too, or + // an order change), position fully: a browse that calls GoTo() for every row it paints + // then gets a page with the neighbouring rows instead of one query per row. + local found as logic + if SELF:_tableMode == TableMode.Table .and. SELF:_recnoColumNo >= 0 .and. SELF:DataTable != null ; + .and. (SELF:_bufferContiguous .or. SELF:RowCount > 1) + found := SELF:_LoadSingleRecord(nRec) + if !found + found := SELF:_GotoRecord(nRec) + endif + else + found := SELF:_GotoRecord(nRec) + endif SELF:_CheckEofBof() RETURN found end method From 88083310d0834d8a298165292b7ff747e601c647 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 11:06:35 +0100 Subject: [PATCH 3/7] SQLRDD: stay on EOF after an order change without loading a page When the work area was at EOF at an order change (typically switching back to the previous order after a seek that found nothing, before going back to the saved record), the reposition did GoTo(0). After the cursor reset that first loaded the first page of the new order and counted it. The order entry form of our ERP did this 4 times per paging step on a 150 column table (~66 ms). An empty result now provides the structure for the phantom record; EOF is marked as pending as with GoTo(0), so Skip(1) stays at EOF and Skip(-1) goes to the bottom. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 92d398ad81..7c4b0d8066 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -1302,8 +1302,19 @@ partial class SQLRDD /// ROW_NUMBER() over the whole order plus a page fetch at a large OFFSET for every switch. /// PRIVATE METHOD _RepositionAfterOrderChange(nRec as DWORD, lWasEof as LOGIC) AS VOID - if SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo == -1 .or. lWasEof + if SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo == -1 SELF:GoTo(iif(lWasEof, 0, nRec)) + elseif lWasEof + // EOF stays EOF (e.g. switching back to the previous order after a seek that found + // nothing, before going back to the saved record). GoTo(0) would first load the + // first page of the new order; an empty result is enough to stand on the phantom + // record. _GoToPhantom() marks the EOF as pending (the empty buffer does not end at + // the last record), so Skip(1) stays at EOF and Skip(-1) goes to the bottom. + if SELF:_OpenTable("1=0", "") + SELF:_GoToPhantom() + else + SELF:GoTo(0) + endif elseif !SELF:_LoadSingleRecord(nRec) SELF:GoTo(nRec) endif From a91d9ae087f60374d96693b22b2137b26d558941 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 11:27:19 +0100 Subject: [PATCH 4/7] SQLRDD: GoTo() after an order change loads only the record as well The protection for browses (a second GoTo() in a row positions fully) also fired when the single buffered record came from an order change. SetOrder() followed by GoTo(saved record) is the usual way to restore a position, and it then paid ROW_NUMBER() plus a page again. Only a single record loaded by the previous GoTo() itself now counts. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 13 ++++++++----- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index e30dd42d50..1218fe2fb3 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -814,14 +814,17 @@ partial class SQLRDD inherit Workarea // is only needed when something moves from it (Skip(), OrderKeyNo ...), see // _EnsurePositioned(): load just the record by its recno. Looking the position up right // away cost a ROW_NUMBER() over the order plus a whole page around the record. - // When the buffer already is such a single record (the previous GoTo() was lazy too, or - // an order change), position fully: a browse that calls GoTo() for every row it paints - // then gets a page with the neighbouring rows instead of one query per row. + // When the previous GoTo() already loaded just one record this way, position fully: a + // browse that calls GoTo() for every row it paints then gets a page with the neighbouring + // rows instead of one query per row. (A single record left by an order change does not + // count - SetOrder() followed by GoTo(saved record) is the usual restore pattern.) local found as logic if SELF:_tableMode == TableMode.Table .and. SELF:_recnoColumNo >= 0 .and. SELF:DataTable != null ; - .and. (SELF:_bufferContiguous .or. SELF:RowCount > 1) + .and. !(SELF:_lazyGoto .and. !SELF:_bufferContiguous .and. SELF:RowCount <= 1) found := SELF:_LoadSingleRecord(nRec) - if !found + if found + SELF:_lazyGoto := TRUE + else found := SELF:_GotoRecord(nRec) endif else diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 7c4b0d8066..301ff58c0c 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -54,6 +54,8 @@ partial class SQLRDD /// seeks and then skips while the key matches mostly reads only a few records. /// private _readAheadRows := 0 as long + /// TRUE when the single buffered record was loaded by GoTo() without its position. + private _lazyGoto as logic /// /// TRUE after GoTo(0) positioned on the phantom record while the buffer does not end at the /// last record of the order: a backward Skip() then has to start from GoBottom(). @@ -1271,6 +1273,7 @@ partial class SQLRDD /// PRIVATE METHOD _LoadSingleRecord(nRec as DWORD) AS LOGIC try + SELF:_lazyGoto := FALSE SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) SELF:_command:ClearParameters() var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) From fea0681d127e94a0c9656a1bda9699b83b36b8a2 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 11:57:02 +0100 Subject: [PATCH 5/7] SQLRDD: keep the current record across an order change Switching orders around a lookup (SetOrder/Seek/SetOrder) read the current record again after every switch, recounted the table and, at EOF, sent a "WHERE 1=0" query only to get an empty buffer. The record was in the buffer a moment before, so it is now kept (its position in the new order is still looked up only when needed), LastRec() stays valid across the switch and the empty buffer is built from the structure of the previous one. A seek that finds nothing sets BOF itself, so it no longer counts the order to compute it. Co-Authored-By: Claude Opus 5.5 --- src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg | 2 +- .../XSharp.SQLRdd/RDD/SQLRDD-Orders.prg | 20 ++++++-- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 48 ++++++++++++++++--- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index 1218fe2fb3..61ef6ab3c5 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -781,7 +781,7 @@ partial class SQLRDD inherit Workarea IF nRec == 0 // DBF semantics: GoTo(0) positions on the phantom record (EOF) without any I/O. // Browsers do this for every empty row they paint, so it must not cost a query. - SELF:_GoToPhantom() + SELF:_GoToPhantom(TRUE) RETURN TRUE ENDIF SELF:_eofPending := FALSE diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg index 86331e7d89..61f80f81d8 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg @@ -195,6 +195,13 @@ partial class SQLRDD // index/order" pattern used throughout the app, and it must keep flushing even when // the order doesn't actually change. SELF:GoCold() + // The record we stand on (just written by GoCold()) is kept across the order change + // instead of being read again, and LastRec() does not change with the order. + local aCurrentRow := null as object[] + if !lWasEof .and. SELF:_hasData .and. SELF:DataTable != null .and. SELF:RowNumber >= 1 .and. SELF:RowNumber <= SELF:RowCount + aCurrentRow := SELF:CurrentRow:ItemArray + endif + var lTableCountValid := SELF:_tableReccountValid if (orderInfo:Order is long var nOrder0 .and. nOrder0 == 0) .or. ; (orderInfo:Order is string var cOrder0 .and. String.IsNullOrEmpty(cOrder0)) // Order 0 (or an empty order name) means: switch back to the natural/physical @@ -211,8 +218,9 @@ partial class SQLRDD self:_CloseCursor() endif SELF:CurrentOrder := null + self:_tableReccountValid := lTableCountValid if lReset - self:_RepositionAfterOrderChange(currentRecord, lWasEof) + self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow) else self:GoTo(currentRecord) endif @@ -224,6 +232,7 @@ partial class SQLRDD self:_CloseCursor() endif SELF:CurrentOrder := newOrder + self:_tableReccountValid := lTableCountValid result := CurrentOrder != null IF result SELF:CurrentOrder:ClearCache() @@ -232,7 +241,7 @@ partial class SQLRDD // loaded row. _CloseCursor() above cleared it, so without this there would // be nothing to evaluate. if lReset - self:_RepositionAfterOrderChange(currentRecord, lWasEof) + self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow) else self:GoTo(currentRecord) endif @@ -569,7 +578,12 @@ partial class SQLRDD // null instead of an empty table - treat that the same as "no rows found" instead of // crashing on DataTable:Rows below, same fix as GoTo()/_ClearTable() already got. IF (SELF:DataTable == null .or. SELF:DataTable:Rows:Count = 0) .and. !seekInfo.SoftSeek - SELF:GoTo(0) + // BOF is set below, so no need to count the order for it (GoTo(0) would) + IF SELF:DataTable == null + SELF:GoTo(0) + ELSE + SELF:_GoToPhantom(FALSE) + ENDIF SELF:_Found := false SELF:_SetEOF(true) SELF:_SetBOF(false) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 301ff58c0c..28cafa8977 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -788,13 +788,17 @@ partial class SQLRDD end method /// Position on the phantom record (EOF) without reading anything. - private method _GoToPhantom() as void + /// FALSE when the caller sets BOF itself, which saves counting the + /// order for an empty buffer (e.g. a seek that found nothing). + private method _GoToPhantom(lSetBof as logic) as void SELF:_Found := FALSE SELF:_outsideOrder := FALSE SELF:RowNumber := SELF:RowCount + 1 SELF:_SetEOF(TRUE) // BOF too only for an empty table (an empty buffer alone does not mean that) - SELF:_SetBOF(SELF:RowCount == 0 .and. SELF:_OrderRowCount() == 0) + if lSetBof + SELF:_SetBOF(SELF:RowCount == 0 .and. SELF:_OrderRowCount() == 0) + endif // When the buffer already ends at the last record, a backward Skip() from the phantom // row naturally lands on it; otherwise remember to go to the bottom first. SELF:_eofPending := !(SELF:_hasEOF .and. SELF:_bufferContiguous) @@ -1304,25 +1308,57 @@ partial class SQLRDD /// and under DBF/ADS the order switches are free. Positioning eagerly here meant a /// ROW_NUMBER() over the whole order plus a page fetch at a large OFFSET for every switch. /// - PRIVATE METHOD _RepositionAfterOrderChange(nRec as DWORD, lWasEof as LOGIC) AS VOID + PRIVATE METHOD _RepositionAfterOrderChange(nRec as DWORD, lWasEof as LOGIC, aCurrentRow as object[]) AS VOID if SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo == -1 SELF:GoTo(iif(lWasEof, 0, nRec)) elseif lWasEof // EOF stays EOF (e.g. switching back to the previous order after a seek that found // nothing, before going back to the saved record). GoTo(0) would first load the - // first page of the new order; an empty result is enough to stand on the phantom + // first page of the new order; an empty buffer is enough to stand on the phantom // record. _GoToPhantom() marks the EOF as pending (the empty buffer does not end at // the last record), so Skip(1) stays at EOF and Skip(-1) goes to the bottom. - if SELF:_OpenTable("1=0", "") - SELF:_GoToPhantom() + if SELF:_SetBufferRows(null) .or. SELF:_OpenTable("1=0", "") + SELF:_GoToPhantom(TRUE) else SELF:GoTo(0) endif + elseif aCurrentRow != null .and. SELF:_SetBufferRows(aCurrentRow) + // The record we stand on was in the buffer a moment ago (written by GoCold() before the + // order change): keep it instead of reading it again. Its position in the new order is + // looked up when needed, as after _LoadSingleRecord(). + SELF:_bufferContiguous := false elseif !SELF:_LoadSingleRecord(nRec) SELF:GoTo(nRec) endif return + /// + /// Replace the buffer with an empty table (aRow NULL) or one row, built from the structure of + /// the previous buffer without a query. FALSE when that structure is not known yet. + /// + PRIVATE METHOD _SetBufferRows(aRow as object[]) AS LOGIC + if SELF:_tableTemplate == null + return false + endif + var oTable := SELF:_tableTemplate:Clone() + if aRow != null + oTable:LoadDataRow(aRow, true) + endif + SELF:_lazyGoto := FALSE + SELF:_ClearTable() + SELF:DataTable := oTable + SELF:RowNumber := 1 + SELF:_outsideOrder := false + SELF:_hasEOF := false + SELF:_anchorWhere := "" + SELF:_bufferStart := 0 + SELF:_bufferContiguous := true + SELF:_readAheadRows := 0 + SELF:_serverReccountValid := false + SELF:_SetEOF(oTable:Rows:Count == 0) + SELF:_SetBOF(FALSE) + return true + /// /// When the buffer only holds records whose position in the current order is unknown /// (see _LoadSingleRecord()), look that position up for the current record now. From b9d4aabfae9805cd1d8a204b29f74338854ad065 Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 13:30:59 +0100 Subject: [PATCH 6/7] SQLRDD: cheaper keyset conditions and small first reads The keyset condition read every key value of the last buffered row with a subquery. Optimizing those took the server 30-45 ms per statement (executing it 3 ms), and since the text changes with every page the plan was never reused. Plain key columns with ASCII string or integer/decimal values are now written as literals; other values still use the subquery. A sequential read of ADRESSDB (52,000 rows, two column key) went from 3.9 to 2.4 s. GoBottom() and looking up the position of a record now read a small page (PageSize/10) instead of a full one, like a seek already does: code that goes to a record mostly reads only it and its neighbour, and a full page of a wide table costs about 10 ms. Reading on makes the pages grow, and a read-ahead that started small still reaches a skip target in one page. Co-Authored-By: Claude Opus 5.5 --- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 74 +++++++++++++++++-- .../Support/SqlDbTableCommandBuilder.prg | 4 +- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index 28cafa8977..a557eec1b6 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -136,6 +136,9 @@ partial class SQLRDD end property /// Maximum number of rows kept in the local buffer. private property _MaxBufferRows as long => _oTd:PageSize * Math.Max(_oTd:BufferSize, 1) + /// Rows for a first read where only a few records are expected to be needed (after a seek, + /// GoBottom(), looking up the position of a record). Reading on makes the pages grow again. + private property _SmallPageRows as long => Math.Max(1, _oTd:PageSize / 10) internal property DataTable as DataTable get return _table @@ -622,7 +625,7 @@ partial class SQLRDD /// See SqlDbTableCommandBuilder.BuildLastPageStatement for why this exists. private method _FetchLastPage(nTotalRows as long) as logic try - SELF:_command:CommandText := _builder:BuildLastPageStatement() + SELF:_command:CommandText := _builder:BuildLastPageStatement(SELF:_SmallPageRows) SELF:_command:ClearParameters() var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) if newTable == null @@ -688,7 +691,7 @@ partial class SQLRDD SELF:_anchorWhere := sAnchorWhere SELF:_bufferContiguous := true // a hard seek (sLoadWhere narrower than the anchor) mostly continues with a few records - SELF:_readAheadRows := iif(sLoadWhere != sAnchorWhere .and. !String.IsNullOrEmpty(sAnchorWhere), Math.Max(1, _oTd:PageSize / 10), 0) + SELF:_readAheadRows := iif(sLoadWhere != sAnchorWhere .and. !String.IsNullOrEmpty(sAnchorWhere), SELF:_SmallPageRows, 0) // A fresh open/reposition (e.g. from Seek()) must not inherit a stale "no more // rows" flag left over from whatever this buffer was doing before - otherwise // Skip() refuses to fetch the next page and reports EOF even though the new @@ -910,7 +913,11 @@ partial class SQLRDD if cExpr == cRecno aValue:Add(nRecno:ToString()) else - aValue:Add("(SELECT " + cExpr + " FROM " + cTable + " WHERE " + cRecno + " = " + nRecno:ToString() + ")") + var cLiteral := SELF:_KeysetLiteral(lastRow, cExpr) + if cLiteral == null + cLiteral := "(SELECT " + cExpr + " FROM " + cTable + " WHERE " + cRecno + " = " + nRecno:ToString() + ")" + endif + aValue:Add(cLiteral) endif next @@ -936,6 +943,43 @@ partial class SQLRDD return sb:ToString() end method + /// + /// The value of the order item cExpr in oRow as a SQL literal, or NULL when the subquery must + /// be used: cExpr is not a plain column, or the value cannot be written as a literal that + /// compares exactly like the stored value. + /// + /// + /// Every subquery has to be optimized by the server: with one per item, compiling the keyset + /// condition of a two column key took 30-45 ms, executing it 3 ms. Since the text changes with + /// every page the plan is never reused. The buffer holds the values as read from the server, so + /// a literal is exact for strings (ASCII only, so no code page conversion can change them) and + /// integers/decimals. + /// + private method _KeysetLiteral(oRow as DataRow, cExpr as string) as string + var nCol := SELF:DataTable:Columns:IndexOf(cExpr:Trim({'[',']','"','`'})) + if nCol < 0 + return null + endif + switch oRow[nCol] + case cValue as string + foreach var c in cValue + if c > (char) 127 + return null + endif + next + return Functions.XsValueToSqlValue(cValue) + case nValue as long + return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + case nValue as int64 + return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + case nValue as short + return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + case nValue as decimal + return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + end switch + return null + end method + private method _KeysetCompare(cExpr as string, lDesc as logic, cValue as string, lInclusive as logic) as string var cComp := iif(lDesc, " <", " >") + iif(lInclusive, "= ", " ") if lDesc @@ -972,6 +1016,9 @@ partial class SQLRDD nCount := iif(nWanted <= nCount, nWanted, Math.Max(nCount, Math.Min(nWanted, nMax))) newTable := SELF:_ReadTable(cWhere, 0, nCount) else + if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows < nCount + nCount := SELF:_readAheadRows + endif newTable := SELF:_ReadTable(SELF:_anchorWhere, nOffset, nCount) endif if newTable == null @@ -1045,8 +1092,9 @@ partial class SQLRDD /// /// Replace the buffer with up to one page of the anchored result set starting at nOffset. /// - private method _LoadBufferAt(nOffset as long) as long - SELF:_readAheadRows := 0 + /// Rows to read, 0 = PageSize. + private method _LoadBufferAt(nOffset as long, nRows := 0 as long) as long + SELF:_readAheadRows := nRows SELF:_ClearTable() SELF:_bufferStart := Math.Max(0, nOffset) SELF:_bufferContiguous := true @@ -1096,6 +1144,10 @@ partial class SQLRDD nNewRow := 1 exit endif + // a read-ahead that started small must still reach the target in one page + if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows < nNewRow - SELF:RowCount + SELF:_readAheadRows := nNewRow - SELF:RowCount + endif if SELF:_FetchForward() == 0 SELF:_hasEOF := true endif @@ -1240,16 +1292,22 @@ partial class SQLRDD if SELF:_bufferContiguous .and. String.IsNullOrEmpty(SELF:_anchorWhere) .and. SELF:RowCount > 0 var nBufferEnd := SELF:_bufferStart + SELF:RowCount if nOffset >= nBufferEnd .and. nOffset < nBufferEnd + _oTd:PageSize .and. !SELF:_hasEOF + // the page must reach the record, a smaller read-ahead would stop short of it + if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows <= nOffset - nBufferEnd + SELF:_readAheadRows := 0 + endif lLoaded := SELF:_FetchForward() > 0 elseif nOffset < SELF:_bufferStart .and. nOffset >= SELF:_bufferStart - _oTd:PageSize lLoaded := SELF:_FetchBackward() > 0 endif endif if !lLoaded - // Center the record in the new page, so that its neighbours on either side - // are buffered as well. + // Center the record in a small page, so that its neighbours on either side are + // buffered as well. Code that goes to a record mostly reads only it and the next + // or previous one (a full page of a wide table costs about 10 ms); reading on makes + // the pages grow. SELF:_anchorWhere := "" - SELF:_LoadBufferAt(nOffset - _oTd:PageSize / 2) + SELF:_LoadBufferAt(nOffset - SELF:_SmallPageRows / 2, SELF:_SmallPageRows) endif endif var nRow := SELF:_FindRowInBuffer(nRec) diff --git a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg index 9ef55a60fb..17a8ceb3c1 100644 --- a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg +++ b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg @@ -223,7 +223,7 @@ internal class SqlDbTableCommandBuilder /// OFFSET 0 is always cheap regardless of table size; the caller is responsible for /// reversing the returned rows back into ascending order. /// - method BuildLastPageStatement() as string + method BuildLastPageStatement(nRows := 0 as long) as string var sb := System.Text.StringBuilder{} local scopeWhere := null as string var CurrentOrder := _oRdd:CurrentOrder @@ -277,7 +277,7 @@ internal class SqlDbTableCommandBuilder sb:Replace(SqlDbProvider.ColumnsMacro, cOrderby) sb:Append(Provider:PagingClause) - sb:Replace(SqlDbProvider.PagesizeMacro, _oTable:PageSize:ToString()) + sb:Replace(SqlDbProvider.PagesizeMacro, iif(nRows > 0, nRows, _oTable:PageSize):ToString()) sb:Replace(SqlDbProvider.StartRecMacro, "0") return sb:ToString() From eb225987cb633a62020cf9c1151015e792c84f7e Mon Sep 17 00:00:00 2001 From: ecosSystem Date: Mon, 26 Oct 2026 14:16:34 +0100 Subject: [PATCH 7/7] SQLRDD: find the window after GoTo and earlier rows by key Positioning on a record (after a lazy GoTo, Skip() from it) loaded a page around it by OFFSET, which makes the server read through all rows before it: 10 ms for a 10 row page at offset 500 of a 1000 row table, 2 ms when the rows are found by key. Reading backwards had the same cost on every page. The page now starts at the record and is found by its key (keyset condition including the record). When its position is not known yet, the buffer is anchored at the record and the position is only looked up when needed. Reading backwards finds the preceding rows by the key of the first buffered row, sorted in reverse, also on a seek-anchored buffer (which is anchored at its new first row), and starts small after a GoTo or GoBottom. Both only use literal key values: an order on a key expression keeps using offsets, because a keyset subquery costs more to compile than the OFFSET. Co-Authored-By: Claude Opus 5.5 --- .../XSharp.SQLRdd/RDD/SQLRDD-Private.prg | 231 +++++++++++++++--- .../Support/SqlDbTableCommandBuilder.prg | 21 +- 2 files changed, 220 insertions(+), 32 deletions(-) diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index a557eec1b6..336b6fabc5 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -54,6 +54,8 @@ partial class SQLRDD /// seeks and then skips while the key matches mostly reads only a few records. /// private _readAheadRows := 0 as long + /// Rows to fetch with the next backward page (0 = PageSize), grows like _readAheadRows. + private _readBackRows := 0 as long /// TRUE when the single buffered record was loaded by GoTo() without its position. private _lazyGoto as logic /// @@ -644,6 +646,8 @@ partial class SQLRDD SELF:_anchorWhere := "" SELF:_bufferContiguous := true SELF:_bufferStart := Math.Max(0, nTotalRows - SELF:RowCount) + // reading backwards from the bottom starts small as well + SELF:_readBackRows := SELF:_SmallPageRows // This IS the last page by definition - without this, a Skip(1) right after // GoBottom() (e.g. GoTo(0)'s GoBottom()+Skip(1)) would not know it's already at // the end, and would fall through to fetching "the next page" via the normal @@ -668,9 +672,9 @@ partial class SQLRDD return self:_OpenTable("", "") end method - private method _ReadTable(sWhereClause as string, nOffset as long, nCount as long) as DataTable + private method _ReadTable(sWhereClause as string, nOffset as long, nCount as long, lReverse := false as logic) as DataTable try - _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount) + _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount, lReverse) _command:ClearParameters() var oDataTable := _command:GetDataTable(self:Alias, SELF:_tableTemplate) return oDataTable @@ -692,6 +696,7 @@ partial class SQLRDD SELF:_bufferContiguous := true // a hard seek (sLoadWhere narrower than the anchor) mostly continues with a few records SELF:_readAheadRows := iif(sLoadWhere != sAnchorWhere .and. !String.IsNullOrEmpty(sAnchorWhere), SELF:_SmallPageRows, 0) + SELF:_readBackRows := 0 // A fresh open/reposition (e.g. from Seek()) must not inherit a stale "no more // rows" flag left over from whatever this buffer was doing before - otherwise // Skip() refuses to fetch the next page and reports EOF even though the new @@ -722,10 +727,10 @@ partial class SQLRDD end method - private method _BuildSqlStatement(sWhereClause as string, nOffset as long, nCount as long) as string + private method _BuildSqlStatement(sWhereClause as string, nOffset as long, nCount as long, lReverse := false as logic) as string local query as string if self:_tableMode == TableMode.Table - query := _builder:BuildSqlStatement(sWhereClause, (int) nOffset, (int) nCount) + query := _builder:BuildSqlStatement(sWhereClause, (int) nOffset, (int) nCount, lReverse) else query := self:_oTd:SelectStatement endif @@ -867,22 +872,36 @@ partial class SQLRDD /// Condition for the rows that follow the last buffered row in the current order (keyset /// paging), or NULL when that cannot be expressed and the caller must page with OFFSET. /// + private method _KeysetAfterLastRow() as string + if SELF:RowCount == 0 .or. !SELF:_bufferContiguous + return null + endif + return SELF:_KeysetCondition(SELF:DataTable:Rows[SELF:RowCount-1], FALSE, FALSE) + end method + + /// + /// Condition for the rows that come after oRow (lBefore FALSE) or before it (lBefore TRUE) in + /// the current order, including oRow itself when lInclusive is TRUE. NULL when that cannot be + /// expressed and the caller must use row offsets instead. + /// /// /// The order is the ORDER BY list of the current order plus the recno column, exactly as /// SqlDbTableCommandBuilder.BuildSqlStatement() sorts. Every item is compared with its value - /// for the last row, read by a subquery on that row's recno, so the comparison uses the same - /// expressions and collation as the ORDER BY (also for keys with functions): + /// for oRow (see _KeysetLiteral(), otherwise read by a subquery on the row's recno, so the + /// comparison uses the same expressions and collation as the ORDER BY, also for keys with + /// functions; lLiteralOnly returns NULL instead, see _KeysetLiteral() for the cost). For the + /// rows after oRow: /// k1 >= v1 AND ( k1 > v1 OR (k1 = v1 AND k2 > v2) OR ... OR (k1 = v1 AND ... AND recno > n) ) - /// The leading bound lets the server seek in an index on k1. A descending item compares with - /// "<" and also accepts NULL, which sorts after all values in descending order. When the last - /// row has NULL in one of the key columns its position cannot be expressed this way. + /// The leading bound lets the server seek in an index on k1. Where the order goes towards + /// lower values (a descending item after oRow, an ascending one before it) the comparison is + /// "<" and also accepts NULL, which sorts below all values. When oRow has NULL in one of the + /// key columns its position cannot be expressed this way. /// - private method _KeysetAfterLastRow() as string - if !SELF:_oTd:HasRecnoColumn .or. SELF:RowCount == 0 .or. !SELF:_bufferContiguous .or. SELF:_recnoColumNo < 0 + private method _KeysetCondition(oRow as DataRow, lBefore as logic, lInclusive as logic, lLiteralOnly := false as logic) as string + if !SELF:_oTd:HasRecnoColumn .or. SELF:_recnoColumNo < 0 return null endif - var lastRow := SELF:DataTable:Rows[SELF:RowCount-1] - var oRecno := lastRow[(int) SELF:_recnoColumNo] + var oRecno := oRow[(int) SELF:_recnoColumNo] if oRecno == null .or. oRecno == DBNull.Value return null endif @@ -895,7 +914,7 @@ partial class SQLRDD if oOrder != null foreach var cCol in oOrder:ColumnList var nCol := SELF:DataTable:Columns:IndexOf(cCol:Trim({'[',']','"','`'})) - if nCol < 0 .or. lastRow[nCol] == DBNull.Value + if nCol < 0 .or. oRow[nCol] == DBNull.Value return null endif next @@ -913,22 +932,26 @@ partial class SQLRDD if cExpr == cRecno aValue:Add(nRecno:ToString()) else - var cLiteral := SELF:_KeysetLiteral(lastRow, cExpr) + var cLiteral := SELF:_KeysetLiteral(oRow, cExpr) if cLiteral == null + if lLiteralOnly + return null + endif cLiteral := "(SELECT " + cExpr + " FROM " + cTable + " WHERE " + cRecno + " = " + nRecno:ToString() + ")" endif aValue:Add(cLiteral) endif next + var nLast := aExpr:Count - 1 var sb := StringBuilder{} sb:Append("(") if aExpr:Count > 1 - sb:Append(SELF:_KeysetCompare(aExpr[0], aDesc[0], aValue[0], TRUE)) + sb:Append(SELF:_KeysetCompare(aExpr[0], aDesc[0] != lBefore, aValue[0], TRUE)) sb:Append(SqlDbProvider.AndClause) endif sb:Append("(") - for var i := 0 upto aExpr:Count-1 + for var i := 0 upto nLast if i > 0 sb:Append(SqlDbProvider.OrClause) endif @@ -936,7 +959,7 @@ partial class SQLRDD for var j := 0 upto i-1 sb:Append(aExpr[j] + " = " + aValue[j] + SqlDbProvider.AndClause) next - sb:Append(SELF:_KeysetCompare(aExpr[i], aDesc[i], aValue[i], FALSE)) + sb:Append(SELF:_KeysetCompare(aExpr[i], aDesc[i] != lBefore, aValue[i], lInclusive .and. i == nLast)) sb:Append(")") next sb:Append("))") @@ -980,9 +1003,9 @@ partial class SQLRDD return null end method - private method _KeysetCompare(cExpr as string, lDesc as logic, cValue as string, lInclusive as logic) as string - var cComp := iif(lDesc, " <", " >") + iif(lInclusive, "= ", " ") - if lDesc + private method _KeysetCompare(cExpr as string, lLower as logic, cValue as string, lInclusive as logic) as string + var cComp := iif(lLower, " <", " >") + iif(lInclusive, "= ", " ") + if lLower return "(" + cExpr + cComp + cValue + SqlDbProvider.OrClause + cExpr + " IS NULL)" endif return cExpr + cComp + cValue @@ -1060,12 +1083,17 @@ partial class SQLRDD /// buffer, evicting rows from the end when the buffer would grow beyond _MaxBufferRows. /// RowNumber keeps pointing at the same row. Returns the number of rows fetched. /// - private method _FetchBackward() as long + /// Rows that must at least be read (when there are that many). + private method _FetchBackward(nNeeded := 0 as long) as long SELF:_readAheadRows := 0 if self:_tableMode != TableMode.Table .or. SELF:_bufferStart <= 0 return 0 endif - var nStart := Math.Max(0, SELF:_bufferStart - _oTd:PageSize) + var nKeyset := SELF:_FetchBackwardKeyset(nNeeded) + if nKeyset >= 0 + return nKeyset + endif + var nStart := Math.Max(0, SELF:_bufferStart - Math.Max(_oTd:PageSize, nNeeded)) var newTable := SELF:_ReadTable(SELF:_anchorWhere, nStart, SELF:_bufferStart - nStart) if newTable == null return 0 @@ -1080,6 +1108,63 @@ partial class SQLRDD next SELF:_bufferStart := nStart SELF:RowNumber += nRows + SELF:_TrimBufferEnd() + return nRows + + /// + /// Like _FetchBackward(), but finds the rows before the buffer by the key of its first row + /// (sorted in reverse) instead of by their offset, which makes the server read through all + /// rows before them. Works on a seek-anchored buffer too, which is then anchored at its new + /// first row. Returns -1 when the key cannot be expressed (see _KeysetCondition()). + /// + private method _FetchBackwardKeyset(nNeeded as long) as long + if self:_tableMode != TableMode.Table .or. SELF:RowCount == 0 .or. !SELF:_bufferContiguous + return -1 + endif + var cBefore := SELF:_KeysetCondition(SELF:DataTable:Rows[0], TRUE, FALSE, TRUE) + if cBefore == null + return -1 + endif + local nCount := iif(SELF:_readBackRows > 0, SELF:_readBackRows, _oTd:PageSize) as long + nCount := Math.Max(nCount, nNeeded) + var newTable := SELF:_ReadTable(cBefore, 0, nCount, TRUE) + if newTable == null + return -1 + endif + var nRows := newTable:Rows:Count + local cAnchor := "" as string + if nRows == nCount .and. !String.IsNullOrEmpty(SELF:_anchorWhere) + // The rows before the anchor are not part of the anchored result set. + cAnchor := SELF:_KeysetCondition(newTable:Rows[nRows-1], FALSE, TRUE, TRUE) + if cAnchor == null + return -1 + endif + endif + // The rows come nearest first. + for var nRow := 0 upto nRows-1 + var newRow := SELF:DataTable:NewRow() + newRow:ItemArray := newTable:Rows[nRow]:ItemArray + SELF:DataTable:Rows:InsertAt(newRow, 0) + newRow:AcceptChanges() + next + SELF:RowNumber += nRows + SELF:_readAheadRows := 0 + SELF:_readBackRows := iif(nRows < nCount, 0, Math.Min(nCount * 2, SELF:_MaxBufferRows)) + if nRows < nCount + // the buffer now starts at the first row of the order + SELF:_anchorWhere := "" + SELF:_bufferStart := 0 + elseif !String.IsNullOrEmpty(SELF:_anchorWhere) + SELF:_anchorWhere := cAnchor + SELF:_bufferStart := 0 + else + SELF:_bufferStart := Math.Max(0, SELF:_bufferStart - nRows) + endif + SELF:_TrimBufferEnd() + return nRows + + /// Evict rows from the end when the buffer grew beyond _MaxBufferRows. + private method _TrimBufferEnd() as void var nRemove := SELF:RowCount - SELF:_MaxBufferRows if nRemove > 0 for var i := 1 upto nRemove @@ -1087,7 +1172,7 @@ partial class SQLRDD next SELF:_hasEOF := false endif - return nRows + return /// /// Replace the buffer with up to one page of the anchored result set starting at nOffset. @@ -1095,6 +1180,7 @@ partial class SQLRDD /// Rows to read, 0 = PageSize. private method _LoadBufferAt(nOffset as long, nRows := 0 as long) as long SELF:_readAheadRows := nRows + SELF:_readBackRows := nRows SELF:_ClearTable() SELF:_bufferStart := Math.Max(0, nOffset) SELF:_bufferContiguous := true @@ -1160,6 +1246,18 @@ partial class SQLRDD SELF:_SetBOF(TRUE) return true endif + if !String.IsNullOrEmpty(SELF:_anchorWhere) .and. 1 - nNewRow <= _oTd:PageSize + // Read the rows before an anchored buffer by key, without looking up its position. + var nFetched := SELF:_FetchBackwardKeyset(1 - nNewRow) + if nFetched == 0 + SELF:RowNumber := 1 + SELF:_SetBOF(TRUE) + return true + elseif nFetched > 0 + nNewRow += nFetched + loop + endif + endif if !String.IsNullOrEmpty(SELF:_anchorWhere) // Seek-anchored buffer: the rows before the anchor are not part of the anchored // result set, so switch to absolute positions first. @@ -1193,12 +1291,13 @@ partial class SQLRDD nNewRow := 1 exit endif - if SELF:_FetchBackward() == 0 + var nRows := SELF:_FetchBackward(1 - nNewRow) + if nRows == 0 SELF:RowNumber := 1 SELF:_SetBOF(TRUE) return true endif - nNewRow := nTarget - SELF:_bufferStart + 1 + nNewRow += nRows enddo SELF:RowNumber := nNewRow SELF:_SetEOF(FALSE) @@ -1270,6 +1369,13 @@ partial class SQLRDD return true PRIVATE METHOD _GotoRecord(nRec as DWORD) AS LOGIC + if SELF:_tableMode == TableMode.Table .and. ; + !(SELF:_bufferContiguous .and. String.IsNullOrEmpty(SELF:_anchorWhere) .and. SELF:RowCount > 0) ; + .and. SELF:_LoadWindowAt(nRec, -1) + // Nothing that could be extended: the buffer now starts at the record, anchored at its + // key, and its position in the order is only looked up when needed. + RETURN TRUE + endif // GoTo() only calls _GotoRecord() after it already established that nRec is NOT in // the currently loaded buffer, so its position in the order must be looked up. var nOffset := SELF:_GetAbsoluteOffset(nRec) @@ -1298,14 +1404,12 @@ partial class SQLRDD endif lLoaded := SELF:_FetchForward() > 0 elseif nOffset < SELF:_bufferStart .and. nOffset >= SELF:_bufferStart - _oTd:PageSize - lLoaded := SELF:_FetchBackward() > 0 + lLoaded := SELF:_FetchBackward(SELF:_bufferStart - nOffset) > 0 endif endif - if !lLoaded + if !lLoaded .and. !SELF:_LoadWindowAt(nRec, nOffset) // Center the record in a small page, so that its neighbours on either side are - // buffered as well. Code that goes to a record mostly reads only it and the next - // or previous one (a full page of a wide table costs about 10 ms); reading on makes - // the pages grow. + // buffered as well. SELF:_anchorWhere := "" SELF:_LoadBufferAt(nOffset - SELF:_SmallPageRows / 2, SELF:_SmallPageRows) endif @@ -1319,6 +1423,70 @@ partial class SQLRDD SELF:_SetBOF(FALSE) RETURN TRUE + /// + /// Replace the buffer with a small page that starts at record nRec, found by its key in the + /// current order (see _KeysetCondition()) instead of its offset: with OFFSET the server reads + /// through all rows before it (10 ms for a 10 row page at offset 500 of a 1000 row table, 2 ms + /// by key). Code that goes to a record mostly reads only it and the next or previous one; + /// reading on makes the pages grow, in both directions. + /// + /// The record's 0 based offset in the order when known, otherwise -1: + /// the buffer is then anchored at the record. + /// FALSE when the key cannot be expressed or the record is not part of the order. + PRIVATE METHOD _LoadWindowAt(nRec as DWORD, nOffset as long) AS LOGIC + local oRow as DataRow + if SELF:CurrentOrder != null + // only orders on plain columns, a key expression has no literal value + foreach var cItem in SELF:CurrentOrder:OrderList + var cExpr := iif(cItem:EndsWith(" DESC", StringComparison.OrdinalIgnoreCase), cItem:Substring(0, cItem:Length - 5), cItem) + if SELF:DataTable == null .or. SELF:DataTable:Columns:IndexOf(cExpr:Trim():Trim({'[',']','"','`'})) < 0 + return false + endif + next + endif + var nRow := SELF:_FindRowInBuffer(nRec) + if nRow > 0 + oRow := SELF:DataTable:Rows[nRow-1] + else + SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec) + SELF:_command:ClearParameters() + var oRecTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate) + if oRecTable == null .or. oRecTable:Rows:Count == 0 + return false + endif + oRow := oRecTable:Rows[0] + endif + var cFrom := SELF:_KeysetCondition(oRow, FALSE, TRUE, TRUE) + if cFrom == null + return false + endif + var nCount := SELF:_SmallPageRows + var newTable := SELF:_ReadTable(cFrom, 0, nCount) + if newTable == null .or. newTable:Rows:Count == 0 .or. Convert.ToUInt32(newTable:Rows[0][self:_recnoColumNo]) != nRec + // not part of the order (FOR condition, scope) + return false + endif + var nRows := newTable:Rows:Count + SELF:_lazyGoto := FALSE + SELF:_ClearTable() + SELF:DataTable := newTable + SELF:RowNumber := 1 + SELF:_outsideOrder := false + SELF:_bufferContiguous := true + if nOffset >= 0 + SELF:_anchorWhere := "" + SELF:_bufferStart := nOffset + else + SELF:_anchorWhere := cFrom + SELF:_bufferStart := 0 + endif + SELF:_hasEOF := nRows < nCount + SELF:_readAheadRows := iif(nRows < nCount, 0, nCount * 2) + SELF:_readBackRows := nCount + SELF:_SetEOF(FALSE) + SELF:_SetBOF(FALSE) + return true + PRIVATE METHOD _GotoRecordOutsideOrder(nRec as DWORD) AS LOGIC if !SELF:_LoadSingleRecord(nRec) // Does not exist even physically. @@ -1412,6 +1580,7 @@ partial class SQLRDD SELF:_bufferStart := 0 SELF:_bufferContiguous := true SELF:_readAheadRows := 0 + SELF:_readBackRows := 0 SELF:_serverReccountValid := false SELF:_SetEOF(oTable:Rows:Count == 0) SELF:_SetBOF(FALSE) diff --git a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg index 17a8ceb3c1..538b5fc227 100644 --- a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg +++ b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg @@ -150,7 +150,8 @@ internal class SqlDbTableCommandBuilder /// Build a fetch of nCount rows of the current order/scope/filter (further restricted by /// sWhereClause), starting at the 0-based row offset nOffset within that result. /// - method BuildSqlStatement(sWhereClause as string, nOffset as int, nCount as int) as string + /// TRUE to sort in reverse, for reading the rows before a given one. + method BuildSqlStatement(sWhereClause as string, nOffset as int, nCount as int, lReverse := false as logic) as string var sb := System.Text.StringBuilder{} local scopeWhere := null as string var CurrentOrder := _oRdd:CurrentOrder @@ -197,9 +198,27 @@ internal class SqlDbTableCommandBuilder cOrderby += " DESC" endif endif + if lReverse + // per item, the items can be expressions with commas + var aItems := List{} + foreach var cItem in CurrentOrder:OrderList + if cItem:EndsWith(" DESC", StringComparison.OrdinalIgnoreCase) + aItems:Add(cItem:Substring(0, cItem:Length - 5):TrimEnd()) + else + aItems:Add(cItem + " DESC") + endif + next + if SELF:_oTable:HasRecnoColumn .and. !Functions.List2String(CurrentOrder:OrderList):Contains(Provider:QuoteIdentifier(self:_oTable:RecnoColumn)) + aItems:Add(Provider:QuoteIdentifier(self:_oTable:RecnoColumn) + iif(CurrentOrder:Descending, "", " DESC")) + endif + cOrderby := Functions.List2String(aItems) + endif elseif SELF:_oTable:HasRecnoColumn sb:Append(Provider:OrderByClause) cOrderby := Provider:QuoteIdentifier(self:_oTable:RecnoColumn) + if lReverse + cOrderby += " DESC" + endif endif cOrderby :=_connection:RaiseStringEvent(_connection, SqlRDDEventReason.OrderByClause, _cTable, cOrderby) sb:Replace(SqlDbProvider.ColumnsMacro, cOrderby)