From 92676970616a5163636329e49c4cc825e46d6c7f Mon Sep 17 00:00:00 2001 From: mitchell Date: Sat, 5 May 2018 23:10:51 -0400 Subject: Backport: Add SC_DOCUMENTOPTION_TEXT_LARGE option for documents larger than 2 GigaBytes. This option is provisional and experimental. Backport of changesets 6696:9729ff36c5b1 and 6723:cffe824ab55e. Also added '#include ' to top of src/RESearch.cxx to fix 32-bit build error. --- curses/ScintillaCurses.cxx | 2 +- doc/ScintillaDoc.html | 49 +++++++++++++++++++++++++++++--------- doc/ScintillaHistory.html | 4 ++++ include/Scintilla.h | 4 +++- include/Scintilla.iface | 10 +++++--- src/CellBuffer.cxx | 16 +++++++++++-- src/CellBuffer.h | 5 +++- src/ContractionState.cxx | 7 ++++-- src/ContractionState.h | 2 +- src/Document.cxx | 11 ++++++--- src/Document.h | 2 ++ src/EditModel.cxx | 2 +- src/EditView.cxx | 7 +++--- src/Editor.cxx | 9 ++++--- src/MarginView.cxx | 13 ++++++---- src/Position.h | 4 ++-- src/RESearch.cxx | 1 + src/RunStyles.cxx | 6 +++++ test/unit/testCellBuffer.cxx | 2 +- test/unit/testContractionState.cxx | 2 +- test/unit/testPartitioning.cxx | 2 +- 21 files changed, 117 insertions(+), 43 deletions(-) diff --git a/curses/ScintillaCurses.cxx b/curses/ScintillaCurses.cxx index af55a3b62..f4804a1f0 100644 --- a/curses/ScintillaCurses.cxx +++ b/curses/ScintillaCurses.cxx @@ -1024,7 +1024,7 @@ public: * pages. The width is based on the width of the view and the view's scroll * width property. */ - bool ModifyScrollBars(int nMax, int nPage) { + bool ModifyScrollBars(Sci::Line nMax, Sci::Line nPage) { if (!wMain.GetID()) return false; WINDOW *w = GetWINDOW(); int maxy = getmaxy(w), maxx = getmaxx(w); diff --git a/doc/ScintillaDoc.html b/doc/ScintillaDoc.html index b4af40a0d..b55c305b4 100644 --- a/doc/ScintillaDoc.html +++ b/doc/ScintillaDoc.html @@ -116,7 +116,7 @@

Scintilla Documentation

-

Last edited 1 February 2018 NH

+

Last edited 17 April 2018 NH

There is an overview of the internal design of Scintilla.
@@ -5688,11 +5688,12 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ SCI_GETDOCPOINTER → document *
SCI_SETDOCPOINTER(<unused>, document *doc)
- SCI_CREATEDOCUMENT(int bytes, int documentOption) → document *
+ SCI_CREATEDOCUMENT(int bytes, int documentOptions) → document *
SCI_ADDREFDOCUMENT(<unused>, document *doc)
SCI_RELEASEDOCUMENT(<unused>, document *doc)
+ SCI_GETDOCUMENTOPTIONS → int

SCI_GETDOCPOINTER → document *
@@ -5709,7 +5710,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ window.
6. If doc was not 0, its reference count is increased by 1.

-

SCI_CREATEDOCUMENT(int bytes, int documentOption) → document *
+

SCI_CREATEDOCUMENT(int bytes, int documentOptions) → document *
This message creates a new, empty document and returns a pointer to it. This document is not selected into the editor and starts with a reference count of 1. This means that you have ownership of it and must either reduce its reference count by 1 after using @@ -5720,12 +5721,29 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ to allocate once rather than rely on the buffer growing as data is added. If SCI_CREATEDOCUMENT fails then 0 is returned.

-

The documentOption argument +

The documentOptions argument chooses between different document capabilities which affect memory allocation and performance with SC_DOCUMENTOPTION_DEFAULT (0) choosing standard options. - SC_DOCUMENTOPTION_STYLES_NONE (1) stops allocation of memory to style characters + SC_DOCUMENTOPTION_STYLES_NONE (0x1) stops allocation of memory to style characters which saves significant memory, often 40% with the whole document treated as being style 0. - Lexers may still produce visual styling by using indicators.

+ Lexers may still produce visual styling by using indicators. + SC_DOCUMENTOPTION_TEXT_LARGE (0x100) accomodates documents larger than 2 GigaBytes + in 64-bit executables. +

+ +

With SC_DOCUMENTOPTION_STYLES_NONE, lexers are still active and may display + indicators. Some may produce folding information althoough most require lexical styles to correctly determine folding. + Its often more efficient to set the null lexer SCLEX_NULL so no lexer is run. +

+ +

The SC_DOCUMENTOPTION_TEXT_LARGE option is experimental and has not been + thoroughly tested. Lexers may fail or hang when lexing past 2GB or 4GB. + Applications using this option should be tested to ensure the option works in their circumstances and each lexer + included should also be tested with documents larger than 4GB. + For many applications lexing documents larger than 4GB will be too sluggish so SC_DOCUMENTOPTION_STYLES_NONE + and the null lexer SCLEX_NULL can be used. Another approach is to turn on idle styling with + SCI_SETIDLESTYLING. +

@@ -5746,10 +5764,16 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ - + + + + + + +
SC_DOCUMENTOPTION_STYLES_NONE10x1 Stop allocation of memory for styles and treat all text as style 0.
SC_DOCUMENTOPTION_TEXT_LARGE0x100Allow document to be larger than 2 GB.
@@ -5771,6 +5795,9 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ world spinning in its orbit you must balance each call to SCI_CREATEDOCUMENT or SCI_ADDREFDOCUMENT with a call to SCI_RELEASEDOCUMENT.

+

SCI_GETDOCUMENTOPTIONS → int
+ Returns the options that were used to create the document.

+

Background loading and saving

To ensure a responsive user interface, applications may decide to load and save documents using a separate thread @@ -5778,7 +5805,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){

Loading in the background

- SCI_CREATELOADER(int bytes, int documentOption) → int
+ SCI_CREATELOADER(int bytes, int documentOptions) → int

An application can load all of a file into a buffer it allocates on a background thread and then add the data in that buffer @@ -5787,15 +5814,15 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){

To avoid these issues, a loader object may be created and used to load the file. The loader object supports the ILoader interface.

-

SCI_CREATELOADER(int bytes, int documentOption) → int
+

SCI_CREATELOADER(int bytes, int documentOptions) → int
Create an object that supports the ILoader interface which can be used to load data and then be turned into a Scintilla document object for attachment to a view object. The bytes argument determines the initial memory allocation for the document as it is more efficient to allocate once rather than rely on the buffer growing as data is added. If SCI_CREATELOADER fails then 0 is returned.

-

The documentOption argument - is described in the SCI_CREATEDOCUMENT section.

+

The documentOptions argument + is described in the SCI_CREATEDOCUMENT section.

ILoader

diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html index 6e41fbb67..61d3e3bf3 100644 --- a/doc/ScintillaHistory.html +++ b/doc/ScintillaHistory.html @@ -539,6 +539,10 @@ Released ?.
  • + Add experimental SC_DOCUMENTOPTION_TEXT_LARGE option to accomodate documents larger than + 2 GigaBytes. +
    • +
  • Set the last X chosen when SCI_REPLACESEL called to ensure macros work when text insertion followed by caret up or down.
    • diff --git a/include/Scintilla.h b/include/Scintilla.h index 69105b70e..d17a53718 100644 --- a/include/Scintilla.h +++ b/include/Scintilla.h @@ -687,10 +687,12 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCI_SETZOOM 2373 #define SCI_GETZOOM 2374 #define SC_DOCUMENTOPTION_DEFAULT 0 -#define SC_DOCUMENTOPTION_STYLES_NONE 1 +#define SC_DOCUMENTOPTION_STYLES_NONE 0x1 +#define SC_DOCUMENTOPTION_TEXT_LARGE 0x100 #define SCI_CREATEDOCUMENT 2375 #define SCI_ADDREFDOCUMENT 2376 #define SCI_RELEASEDOCUMENT 2377 +#define SCI_GETDOCUMENTOPTIONS 2379 #define SCI_GETMODEVENTMASK 2378 #define SCI_SETFOCUS 2380 #define SCI_GETFOCUS 2381 diff --git a/include/Scintilla.iface b/include/Scintilla.iface index d2bca177c..e7c6d4725 100644 --- a/include/Scintilla.iface +++ b/include/Scintilla.iface @@ -1780,16 +1780,20 @@ get int GetZoom=2374(,) enu DocumentOption=SC_DOCUMENTOPTION_ val SC_DOCUMENTOPTION_DEFAULT=0 -val SC_DOCUMENTOPTION_STYLES_NONE=1 +val SC_DOCUMENTOPTION_STYLES_NONE=0x1 +val SC_DOCUMENTOPTION_TEXT_LARGE=0x100 # Create a new document object. # Starts with reference count of 1 and not selected into editor. -fun int CreateDocument=2375(int bytes, int documentOption) +fun int CreateDocument=2375(int bytes, int documentOptions) # Extend life of document. fun void AddRefDocument=2376(, int doc) # Release a reference to the document, deleting document if it fades to black. fun void ReleaseDocument=2377(, int doc) +# Get which document options are set. +get int GetDocumentOptions=2379(,) + # Get which document modification events are sent to the container. get int GetModEventMask=2378(,) @@ -2551,7 +2555,7 @@ set void SetTechnology=2630(int technology,) get int GetTechnology=2631(,) # Create an ILoader*. -fun int CreateLoader=2632(int bytes, int documentOption) +fun int CreateLoader=2632(int bytes, int documentOptions) # On OS X, show a find indicator. fun void FindIndicatorShow=2640(position start, position end) diff --git a/src/CellBuffer.cxx b/src/CellBuffer.cxx index b1836b924..41dd36b56 100644 --- a/src/CellBuffer.cxx +++ b/src/CellBuffer.cxx @@ -364,12 +364,16 @@ void UndoHistory::CompletedRedoStep() { currentAction++; } -CellBuffer::CellBuffer(bool hasStyles_) : - hasStyles(hasStyles_) { +CellBuffer::CellBuffer(bool hasStyles_, bool largeDocument_) : + hasStyles(hasStyles_), largeDocument(largeDocument_) { readOnly = false; utf8LineEnds = 0; collectingUndo = true; plv = std::unique_ptr>(new LineVector()); + if (largeDocument) + plv = std::unique_ptr>(new LineVector()); + else + plv = std::unique_ptr>(new LineVector()); } CellBuffer::~CellBuffer() { @@ -556,6 +560,14 @@ void CellBuffer::SetReadOnly(bool set) { readOnly = set; } +bool CellBuffer::IsLarge() const { + return largeDocument; +} + +bool CellBuffer::HasStyles() const { + return hasStyles; +} + void CellBuffer::SetSavePoint() { uh.SetSavePoint(); } diff --git a/src/CellBuffer.h b/src/CellBuffer.h index 544a26711..935ea6e69 100644 --- a/src/CellBuffer.h +++ b/src/CellBuffer.h @@ -108,6 +108,7 @@ public: class CellBuffer { private: bool hasStyles; + bool largeDocument; SplitVector substance; SplitVector style; bool readOnly; @@ -126,7 +127,7 @@ private: public: - CellBuffer(bool hasStyles_); + CellBuffer(bool hasStyles_, bool largeDocument_); // Deleted so CellBuffer objects can not be copied. CellBuffer(const CellBuffer &) = delete; void operator=(const CellBuffer &) = delete; @@ -163,6 +164,8 @@ public: bool IsReadOnly() const; void SetReadOnly(bool set); + bool IsLarge() const; + bool HasStyles() const; /// The save point is a marker in the undo stack where the container has stated that /// the buffer was saved. Undo and redo can move over the save point. diff --git a/src/ContractionState.cxx b/src/ContractionState.cxx index 5922bf546..be3c3d6ad 100644 --- a/src/ContractionState.cxx +++ b/src/ContractionState.cxx @@ -407,8 +407,11 @@ void ContractionState::Check() const { namespace Scintilla { -std::unique_ptr ContractionStateCreate() { - return std::unique_ptr>(new ContractionState()); +std::unique_ptr ContractionStateCreate(bool largeDocument) { + if (largeDocument) + return std::unique_ptr>(new ContractionState()); + else + return std::unique_ptr>(new ContractionState()); } } diff --git a/src/ContractionState.h b/src/ContractionState.h index a951e0a0c..90f5c0784 100644 --- a/src/ContractionState.h +++ b/src/ContractionState.h @@ -45,7 +45,7 @@ public: virtual void ShowAll()=0; }; -std::unique_ptr ContractionStateCreate(); +std::unique_ptr ContractionStateCreate(bool largeDocument); } diff --git a/src/Document.cxx b/src/Document.cxx index 34238af35..b1128b82b 100644 --- a/src/Document.cxx +++ b/src/Document.cxx @@ -86,7 +86,7 @@ int LexInterface::LineEndTypesSupported() { } Document::Document(int options) : - cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0) { + cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0) { refCount = 0; #ifdef _WIN32 eolMode = SC_EOL_CRLF; @@ -117,7 +117,7 @@ Document::Document(int options) : perLineData[ldMargin].reset(new LineAnnotation()); perLineData[ldAnnotation].reset(new LineAnnotation()); - decorations = DecorationListCreate(false); + decorations = DecorationListCreate(IsLarge()); cb.SetPerLine(this); } @@ -556,7 +556,7 @@ void Document::GetHighlightDelimiters(HighlightDelimiter &highlightDelimiter, Sc } Sci::Position Document::ClampPositionIntoDocument(Sci::Position pos) const { - return Sci::clamp(pos, 0, static_cast(Length())); + return Sci::clamp(pos, static_cast(0), static_cast(Length())); } bool Document::IsCrLf(Sci::Position pos) const { @@ -1524,6 +1524,11 @@ void Document::ConvertLineEnds(int eolModeSet) { } +int Document::Options() const { + return (IsLarge() ? SC_DOCUMENTOPTION_TEXT_LARGE : 0) | + (cb.HasStyles() ? 0 : SC_DOCUMENTOPTION_STYLES_NONE); +} + bool Document::IsWhiteLine(Sci::Line line) const { Sci::Position currentChar = static_cast(LineStart(line)); const Sci::Position endLine = static_cast(LineEnd(line)); diff --git a/src/Document.h b/src/Document.h index ba811b8da..ecb57195d 100644 --- a/src/Document.h +++ b/src/Document.h @@ -352,6 +352,8 @@ public: void ConvertLineEnds(int eolModeSet); void SetReadOnly(bool set) { cb.SetReadOnly(set); } bool IsReadOnly() const { return cb.IsReadOnly(); } + bool IsLarge() const { return cb.IsLarge(); } + int Options() const; void DelChar(Sci::Position pos); void DelCharBack(Sci::Position pos); diff --git a/src/EditModel.cxx b/src/EditModel.cxx index 53301e428..df9efb689 100644 --- a/src/EditModel.cxx +++ b/src/EditModel.cxx @@ -69,7 +69,7 @@ EditModel::EditModel() { wrapWidth = LineLayout::wrapWidthInfinite; pdoc = new Document(SC_DOCUMENTOPTION_DEFAULT); pdoc->AddRef(); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); } EditModel::~EditModel() { diff --git a/src/EditView.cxx b/src/EditView.cxx index d9ca751a0..5ff43558d 100644 --- a/src/EditView.cxx +++ b/src/EditView.cxx @@ -2330,16 +2330,15 @@ Sci::Position EditView::FormatRange(bool draw, Sci_RangeToFormat *pfr, Surface * if (draw && lineNumberWidth && (ypos + vsPrint.lineHeight <= pfr->rc.bottom) && (visibleLine >= 0)) { - char number[100]; - sprintf(number, "%d" lineNumberPrintSpace, lineDoc + 1); + const std::string number = std::to_string(lineDoc + 1) + lineNumberPrintSpace; PRectangle rcNumber = rcLine; rcNumber.right = rcNumber.left + lineNumberWidth; // Right justify rcNumber.left = rcNumber.right - surfaceMeasure->WidthText( - vsPrint.styles[STYLE_LINENUMBER].font, number, static_cast(strlen(number))); + vsPrint.styles[STYLE_LINENUMBER].font, number.c_str(), static_cast(number.length())); surface->FlushCachedState(); surface->DrawTextNoClip(rcNumber, vsPrint.styles[STYLE_LINENUMBER].font, - static_cast(ypos + vsPrint.maxAscent), number, static_cast(strlen(number)), + static_cast(ypos + vsPrint.maxAscent), number.c_str(), static_cast(number.length()), vsPrint.styles[STYLE_LINENUMBER].fore, vsPrint.styles[STYLE_LINENUMBER].back); } diff --git a/src/Editor.cxx b/src/Editor.cxx index 79fe1addc..3571713dc 100644 --- a/src/Editor.cxx +++ b/src/Editor.cxx @@ -5212,7 +5212,7 @@ void Editor::SetDocPointer(Document *document) { pdoc = document; } pdoc->AddRef(); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); // Ensure all positions within document sel.Clear(); @@ -7595,7 +7595,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { Document *doc = new Document(static_cast(lParam)); doc->AddRef(); doc->Allocate(static_cast(wParam)); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); return reinterpret_cast(doc); } @@ -7607,12 +7607,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { (reinterpret_cast(lParam))->Release(); break; + case SCI_GETDOCUMENTOPTIONS: + return pdoc->Options(); + case SCI_CREATELOADER: { Document *doc = new Document(static_cast(lParam)); doc->AddRef(); doc->Allocate(static_cast(wParam)); doc->SetUndoCollection(false); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); return reinterpret_cast(static_cast(doc)); } diff --git a/src/MarginView.cxx b/src/MarginView.cxx index a7f9a9a2d..02a961b10 100644 --- a/src/MarginView.cxx +++ b/src/MarginView.cxx @@ -367,10 +367,12 @@ void MarginView::PaintMargin(Surface *surface, Sci::Line topLine, PRectangle rc, rcMarker.bottom = static_cast(yposScreen + vs.lineHeight); if (vs.ms[margin].style == SC_MARGIN_NUMBER) { if (firstSubLine) { - char number[100] = ""; - if (lineDoc >= 0) - sprintf(number, "%d", lineDoc + 1); + std::string sNumber; + if (lineDoc >= 0) { + sNumber = std::to_string(lineDoc + 1); + } if (model.foldFlags & (SC_FOLDFLAG_LEVELNUMBERS | SC_FOLDFLAG_LINESTATE)) { + char number[100] = ""; if (model.foldFlags & SC_FOLDFLAG_LEVELNUMBERS) { const int lev = model.pdoc->GetLevel(lineDoc); sprintf(number, "%c%c %03X %03X", @@ -383,14 +385,15 @@ void MarginView::PaintMargin(Surface *surface, Sci::Line topLine, PRectangle rc, const int state = model.pdoc->GetLineState(lineDoc); sprintf(number, "%0X", state); } + sNumber = number; } PRectangle rcNumber = rcMarker; // Right justify - const XYPOSITION width = surface->WidthText(fontLineNumber, number, static_cast(strlen(number))); + const XYPOSITION width = surface->WidthText(fontLineNumber, sNumber.c_str(), static_cast(sNumber.length())); const XYPOSITION xpos = rcNumber.right - width - vs.marginNumberPadding; rcNumber.left = xpos; DrawTextNoClipPhase(surface, rcNumber, vs.styles[STYLE_LINENUMBER], - rcNumber.top + vs.maxAscent, number, static_cast(strlen(number)), drawAll); + rcNumber.top + vs.maxAscent, sNumber.c_str(), static_cast(sNumber.length()), drawAll); } else if (vs.wrapVisualFlags & SC_WRAPVISUALFLAG_MARGIN) { PRectangle rcWrapMarker = rcMarker; rcWrapMarker.right -= wrapMarkerPaddingRight; diff --git a/src/Position.h b/src/Position.h index 804d8681a..ab8223b54 100644 --- a/src/Position.h +++ b/src/Position.h @@ -16,8 +16,8 @@ namespace Sci { -typedef int Position; -typedef int Line; +typedef ptrdiff_t Position; +typedef ptrdiff_t Line; const Position invalidPosition = -1; diff --git a/src/RESearch.cxx b/src/RESearch.cxx index f7ae04974..85a7171f4 100644 --- a/src/RESearch.cxx +++ b/src/RESearch.cxx @@ -200,6 +200,7 @@ * matches: foo-foo fo-fo fob-fob foobar-foobar ... */ +#include #include #include diff --git a/src/RunStyles.cxx b/src/RunStyles.cxx index 620422dc8..6309edc44 100644 --- a/src/RunStyles.cxx +++ b/src/RunStyles.cxx @@ -6,9 +6,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -306,3 +308,7 @@ void RunStyles::Check() const { template class Scintilla::RunStyles; template class Scintilla::RunStyles; +#if PTRDIFF_MAX != INT_MAX +template class Scintilla::RunStyles; +template class Scintilla::RunStyles; +#endif diff --git a/test/unit/testCellBuffer.cxx b/test/unit/testCellBuffer.cxx index f0192423c..cef88cb17 100644 --- a/test/unit/testCellBuffer.cxx +++ b/test/unit/testCellBuffer.cxx @@ -26,7 +26,7 @@ TEST_CASE("CellBuffer") { const char sText[] = "Scintilla"; const Sci::Position sLength = static_cast(strlen(sText)); - CellBuffer cb(true); + CellBuffer cb(true, false); SECTION("InsertOneLine") { bool startSequence = false; diff --git a/test/unit/testContractionState.cxx b/test/unit/testContractionState.cxx index b0346dfc9..fcee90287 100644 --- a/test/unit/testContractionState.cxx +++ b/test/unit/testContractionState.cxx @@ -25,7 +25,7 @@ using namespace Scintilla; TEST_CASE("ContractionState") { - std::unique_ptr pcs = ContractionStateCreate(); + std::unique_ptr pcs = ContractionStateCreate(false); SECTION("IsEmptyInitially") { REQUIRE(1 == pcs->LinesInDoc()); diff --git a/test/unit/testPartitioning.cxx b/test/unit/testPartitioning.cxx index 69bbe7957..449253ef3 100644 --- a/test/unit/testPartitioning.cxx +++ b/test/unit/testPartitioning.cxx @@ -50,7 +50,7 @@ TEST_CASE("SplitVectorWithRangeAdd") { TEST_CASE("Partitioning") { - Partitioning part(growSize); + Partitioning part(growSize); SECTION("IsEmptyInitially") { REQUIRE(1 == part.Partitions()); -- cgit v1.2.3