SQLRDD: large tables - date and non-ASCII keyset literals, direct SeekLast and GoBottom, no deep OFFSET, OPTION (RECOMPILE) for keysets

src/Runtime/XSharp.SQLRdd/Classes/Connection.prg

@@ -539,6 +539,70 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         return result
     end method
 
+    private _literalEncoding as System.Text.Encoding
+    private _literalEncodingRead as logic
+    /// <summary>
+    /// TRUE when cValue can be written as a string literal without a character being changed: ASCII,
+    /// or all characters in the code page of the database (see SqlDbProvider.CodePageStatement, read
+    /// once per connection).
+    /// </summary>
+    internal method IsExactStringLiteral(cValue as string) as logic
+        local lAscii := true as logic
+        foreach var c in cValue
+            if c > (char) 127
+                lAscii := false
+                exit
+            endif
+        next
+        if lAscii
+            return true
+        endif
+        if !_literalEncodingRead
+            _literalEncodingRead := true
+            if SELF:Provider is SqlDbProvider var oProvider .and. !String.IsNullOrEmpty(oProvider:CodePageStatement)
+                try
+                    var oResult := SELF:ExecuteScalar(oProvider:CodePageStatement)
+                    var nCodePage := iif(oResult == null .or. oResult == DBNull.Value, 0, Convert.ToInt32(oResult))
+                    if nCodePage > 0
+                        _literalEncoding := System.Text.Encoding.GetEncoding(nCodePage, System.Text.EncoderFallback.ExceptionFallback, System.Text.DecoderFallback.ExceptionFallback)
+                    endif
+                catch
+                    _literalEncoding := null
+                end try
+            endif
+        endif
+        if _literalEncoding == null
+            return false
+        endif
+        try
+            return _literalEncoding:GetString(_literalEncoding:GetBytes(cValue)) == cValue
+        catch
+            return false
+        end try
+    end method
+
+    private _tableRows := Dictionary<string, int64>{StringComparer.OrdinalIgnoreCase} as Dictionary<string, int64>
+    /// <summary>Approximate number of rows of a table from the statistics of the database (see
+    /// SqlDbProvider.TableRowsStatement), read once per connection; -1 when unknown.</summary>
+    internal method ApproximateRows(cTable as string) as int64
+        local nRows as int64
+        if String.IsNullOrEmpty(cTable)
+            return -1
+        endif
+        if _tableRows:TryGetValue(cTable, out nRows)
+            return nRows
+        endif
+        nRows := -1
+        if SELF:Provider is SqlDbProvider var oProvider .and. !String.IsNullOrEmpty(oProvider:TableRowsStatement)
+            var oResult := SELF:ExecuteScalar(oProvider:TableRowsStatement:Replace(SqlDbProvider.TableNameMacro, cTable:Replace("'", "''")), cTable)
+            if oResult != null .and. oResult != DBNull.Value
+                nRows := Convert.ToInt64(oResult)
+            endif
+        endif
+        _tableRows[cTable] := nRows
+        return nRows
+    end method
+
    /// <summary>
    /// Execute a SQL command
    /// </summary>

src/Runtime/XSharp.SQLRdd/DBMS/DbProvider.prg

@@ -231,6 +231,40 @@ abstract class SqlDbProvider inherit SqlDbObject implements ISqlDbProvider
         return cPattern
     end method
 
+    /// <summary>
+    /// A date without a time of day as a SQL literal that compares exactly like the stored value, or
+    /// NULL when there is no such literal (the RDD then reads the value with a subquery).
+    /// The default implementation returns NULL: how a string converts to a date depends on the
+    /// database and the settings of the session.
+    /// </summary>
+    virtual method DateLiteral(dValue as DateTime) as string
+        return null
+    end method
+
+    /// <summary>
+    /// Appended to a query of the RDD whose condition has OR, LIKE or range terms (the keyset condition
+    /// of a page, a soft seek, a scope), on a table with at least RangeQueryHintMinRows rows. The default
+    /// implementation returns an empty string.
+    /// </summary>
+    virtual property RangeQueryHint as string => ""
+
+    /// <summary>
+    /// Statement that returns the code page in which the database stores the string literals of a
+    /// statement (characters outside it are changed). The default implementation returns an empty
+    /// string (unknown: the RDD only writes ASCII strings as literals).
+    /// </summary>
+    virtual property CodePageStatement as string => ""
+
+    /// <summary>Minimum number of rows of a table (see TableRowsStatement) for RangeQueryHint.</summary>
+    virtual property RangeQueryHintMinRows as long => 100000
+
+    /// <summary>
+    /// Statement that returns the approximate number of rows of the table TableNameMacro from the
+    /// statistics of the database, without counting them. The default implementation returns an
+    /// empty string (unknown).
+    /// </summary>
+    virtual property TableRowsStatement as string => ""
+
     /// <summary>
     /// Create a new SqlDbProvider object
     /// </summary>

src/Runtime/XSharp.SQLRdd/DBMS/SqlServer.prg

@@ -58,6 +58,38 @@ class SqlDbProviderSqlServer inherit SqlDbProvider
     /// <summary>SQL Server also reads [...] in a LIKE pattern as a character class.</summary>
     override property LikeWildcards as string => "%_["
 
+    /// <inheritdoc />
+    /// <remarks>'yyyyMMdd' is read the same way for date, datetime, datetime2 and smalldatetime,
+    /// whatever the language and DATEFORMAT of the session.</remarks>
+    override method DateLiteral(dValue as DateTime) as string
+        return "'" + dValue:ToString("yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture) + "'"
+    end method
+
+    /// <inheritdoc />
+    /// <remarks>
+    /// A keyset condition k1 >= v1 AND (k1 > v1 OR (k1 = v1 AND k2 > v2) OR ...) can only be turned
+    /// into seeks on the index when the server knows the values. With PARAMETERIZATION FORCED (or a
+    /// plan from the cache) it does not: it seeks on k1 alone and reads all rows of v1 before the
+    /// position. Paging the lines of an order near the end of a table with 5.4 million rows read
+    /// 6,641 pages (860 ms CPU) for 10 rows, with OPTION (RECOMPILE) 54 pages (2 ms to compile).
+    /// A range is at risk as well: a plan from the cache for TYP = @0 AND KURZNAME >= @1 ORDER BY
+    /// KURZNAME, made for the 1.1 million customers (TYP 'K'), read the short name index in order and
+    /// looked up TYP for every row. For a soft seek among the 488 suppliers (TYP 'L') near the end of
+    /// the names it read 720,398 pages (107 s) to find nothing; recompiled, 418 pages (6 ms).
+    /// On small tables these reads are cheap and compiling every query costs more: navigation tests on
+    /// tables with 1,000 to 50,000 rows took 1.5 to 2 times as long, hence RangeQueryHintMinRows.
+    /// </remarks>
+    override property RangeQueryHint as string => " OPTION (RECOMPILE)"
+
+    /// <inheritdoc />
+    /// <remarks>A varchar literal takes the default collation of the database.</remarks>
+    override property CodePageStatement as string => ;
+        "SELECT COLLATIONPROPERTY(CONVERT(sysname, DATABASEPROPERTYEX(DB_NAME(), 'Collation')), 'CodePage')"
+
+    /// <inheritdoc />
+    override property TableRowsStatement as string => ;
+        "SELECT SUM(rows) FROM sys.partitions WHERE object_id = OBJECT_ID(N'" + TableNameMacro + "') AND index_id IN (0, 1)"
+
     private static aFuncs := null as Dictionary<string, string>
     /// <inheritdoc />
     override method GetFunctions() as Dictionary<string, string>

src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg

@@ -318,6 +318,8 @@ internal class SqlDbOrder inherit SqlDbObject
         foreach var oTerm in terms
             if oTerm:IsDate
                 widths:Add(8)
+            elseif oTerm:Width > 0
+                widths:Add((int) oTerm:Width)
             else
                 var cName := oTerm:Column:Trim(<char>{'[',']','"','`'})
                 var oCol := self:RDD:TableColumns:FirstOrDefault({ c => String.Compare(c:Name, cName, true) == 0 })
@@ -361,7 +363,7 @@ internal class SqlDbOrder inherit SqlDbObject
                     sb:Append(SqlDbProvider.AndClause)
                 endif
                 if parts[i]:Length >= widths[i]
-                    sb:Append(columns[i] + " = " + Functions.XsValueToSqlValue(parts[i]))
+                    sb:Append(SELF:_PartCompare(terms[i], columns[i], " = ", parts[i]))
                 else
                     sb:Append(columns[i] + " like " + Provider:LikePattern(parts[i]))
                 endif
@@ -379,17 +381,26 @@ internal class SqlDbOrder inherit SqlDbObject
                 endif
                 sb:Append("(")
                 for var j := 0 upto i-1
-                    sb:Append(columns[j] + " = " + Functions.XsValueToSqlValue(parts[j]))
+                    sb:Append(SELF:_PartCompare(terms[j], columns[j], " = ", parts[j]))
                     sb:Append(SqlDbProvider.AndClause)
                 next
-                var cColExpr := columns[i]
+                var cComp := iif(i == parts:Count-1, cRangeComp, cStrictComp)
                 if parts[i]:Length < widths[i]
-                    // Compare only the prefix of the column that the value actually covers.
-                    var cSubstr := Provider:GetFunction("SUBSTR(%1%,%2%,%3%)")
-                    cColExpr := cSubstr:Replace("%1%", cColExpr):Replace("%2%","1"):Replace("%3%", parts[i]:Length:ToString())
+                    // Compare only the prefix of the column that the value actually covers (only
+                    // the last part can be shorter, so this is the inclusive bound). SUBSTRING(c,1,n)
+                    // would keep the server from seeking on the index: a soft seek in a descending
+                    // order of 3.8 million orders took 0.7 to 1.7 seconds. The same conditions:
+                    //   SUBSTRING(c,1,n) >= v   is   c >= v
+                    //   SUBSTRING(c,1,n) <= v   is   c < v OR c LIKE 'v%'
+                    if cRangeComp:Contains(">")
+                        sb:Append(columns[i] + " >= " + Functions.XsValueToSqlValue(parts[i]))
+                    else
+                        sb:Append("(" + columns[i] + " < " + Functions.XsValueToSqlValue(parts[i]) + ;
+                            SqlDbProvider.OrClause + columns[i] + " like " + Provider:LikePattern(parts[i]) + ")")
+                    endif
+                else
+                    sb:Append(SELF:_PartCompare(terms[i], columns[i], cComp, parts[i]))
                 endif
-                var cComp := iif(i == parts:Count-1, cRangeComp, cStrictComp)
-                sb:Append(cColExpr + cComp + Functions.XsValueToSqlValue(parts[i]))
                 sb:Append(")")
             next
             if parts:Count > 1
@@ -400,5 +411,19 @@ internal class SqlDbOrder inherit SqlDbObject
         return sb:ToString()
     end method
 
+    /// <summary>
+    /// cExpr cComp cValue for a key part the value covers completely. A STR() part compares the number,
+    /// it is sorted by the number (see SqlDbExpression.GetKeyTerms()): [MENGE] = 1 instead of
+    /// STR([MENGE],5,0) = '    1', which also lets the server seek on an index with MENGE.
+    /// </summary>
+    private method _PartCompare(oTerm as SqlDbKeyTerm, cExpr as string, cComp as string, cValue as string) as string
+        local nValue as decimal
+        if oTerm:Width > 0 .and. Decimal.TryParse(cValue:Trim(), System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint, ;
+            System.Globalization.CultureInfo.InvariantCulture, out nValue)
+            return oTerm:Column + cComp + nValue:ToString(System.Globalization.CultureInfo.InvariantCulture)
+        endif
+        return cExpr + cComp + Functions.XsValueToSqlValue(cValue)
+    end method
+
 end class
 end namespace // XSharp.RDD.SqlRDD

src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg

@@ -702,25 +702,17 @@ partial class SQLRDD inherit Workarea
         SELF:_outsideOrder := FALSE
         SELF:_eofPending := FALSE
         SELF:_ClearTable()
-        // Route through the cached count instead of always querying directly - GoBottom() can
-        // be called many times in a row against unchanged data (e.g. Lister construction
-        // retrying GoTo(0) when it can't establish a valid position), and each direct query
-        // here used to pay for a fresh COUNT(*) every single time.
-        self:_GetRecCount()
-        local nTotal := (long) self:_serverReccount as long
         SELF:_anchorWhere := ""
-        // For a large table, fetching the last page via the normal ascending, OFFSET-based
-        // query forces the server to walk/skip almost the entire ordered result -
-        // the further from the top, the worse. _FetchLastPage avoids that by sorting in
-        // reverse and asking for OFFSET 0 instead, which is always cheap. Only safe when the
-        // order isn't already descending (that would invert the reversal); falls back to the
-        // original approach for natural order, descending orders, or if it fails outright.
-        if self:CurrentOrder == null .or. !self:CurrentOrder:Descending
-            if !SELF:_FetchLastPage(nTotal)
-                SELF:_LoadBufferAt(SELF:_LastPageStart(nTotal))
-            endif
-        else
-            SELF:_LoadBufferAt(SELF:_LastPageStart(nTotal))
+        // For a large table, fetching the last page via the normal OFFSET-based query forces
+        // the server to walk/skip almost the entire ordered result - the further from the top,
+        // the worse: OFFSET 3768900 in a descending order of 3.8 million orders did not finish
+        // within 30 seconds. _FetchLastPage avoids that by sorting in reverse and asking for
+        // OFFSET 0 instead, which is always cheap. Falls back to the OFFSET only if it fails.
+        // The count is only needed for the OFFSET (see _FetchLastPage()); it goes through the
+        // cached count - GoBottom() can be called many times in a row against unchanged data.
+        if !SELF:_FetchLastPage()
+            self:_GetRecCount()
+            SELF:_LoadBufferAt(SELF:_LastPageStart((long) self:_serverReccount))
         endif
         SELF:RowNumber  := SELF:RowCount
         SELF:_Top       := FALSE

src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg

@@ -566,6 +566,9 @@ partial class SQLRDD
             return false
         endif
         var cSeekWhere := CurrentOrder:SeekExpression(seekInfo )
+        if seekInfo:Last .and. SELF:_SeekLastDirect(seekInfo)
+            return true
+        endif
         // The buffer is anchored at the soft-seek condition for the same value: the rows a
         // hard seek returns are exactly the leading rows of that result, so paging forward past
         // them (the common "seek to the first record of a key, then Skip() while the key still
@@ -652,6 +655,42 @@ partial class SQLRDD
         return true
     end method
 
+    /// <summary>
+    /// SeekLast: read the last record of the key directly, with the order reversed, and position
+    /// on it. FALSE when that is not possible (filter, no recno column) or the key has no record:
+    /// the caller then continues as before (soft seek, not found).
+    /// </summary>
+    /// <remarks>
+    /// Seek() found the first record of the key and skipped from there to the last one. That reads
+    /// every record of the key, page by page: BelegHdr:GoTop() in our ERP seeks the newest order of
+    /// a type with SeekLast on BEL_ART+DTOS(DAT_HINZU)+BEL_NR, and with 3.8 million orders it had
+    /// not arrived after 80 seconds and 310,000 records. The position of the record in the order is
+    /// looked up when something moves from it, as after GoTo().
+    /// </remarks>
+    PRIVATE METHOD _SeekLastDirect(seekInfo as DbSeekInfo) AS LOGIC
+        if SELF:_FilterInfo:Active .or. SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo < 0
+            return false
+        endif
+        // the records of the key also for a soft seek: it only differs when the key has none
+        var hardInfo := seekInfo
+        hardInfo:SoftSeek := FALSE
+        hardInfo:Last     := FALSE
+        var oLast := SELF:_ReadTable(SELF:CurrentOrder:SeekExpression(hardInfo), 0, 1, TRUE)
+        if oLast == null .or. oLast:Rows:Count == 0
+            SELF:_ReturnSpareTable(oLast)
+            return false
+        endif
+        var nRec := Convert.ToUInt32(oLast:Rows[0][SELF:_recnoColumNo])
+        SELF:_ReturnSpareTable(oLast)
+        if !SELF:GoTo(nRec) .or. SELF:_EoF
+            return false
+        endif
+        SELF:_SetEOF(FALSE)
+        SELF:_SetBOF(FALSE)
+        SELF:Found := TRUE
+        return true
+    end method
+
     PRIVATE METHOD KeyCompare(oRecKey as OBJECT, oKey as OBJECT) AS LONG
         LOCAL result  AS INT64
         IF oRecKey IS NULL