1/* Part of SWI-Prolog 2 3 Author: Jan Wielemaker 4 E-mail: jan@swi-prolog.org 5 WWW: https://www.swi-prolog.org 6 Copyright (c) 2009-2025, VU University, Amsterdam 7 CWI, Amsterdam 8 SWI-Prolog Solutions b.v. 9 All rights reserved. 10 11 Redistribution and use in source and binary forms, with or without 12 modification, are permitted provided that the following conditions 13 are met: 14 15 1. Redistributions of source code must retain the above copyright 16 notice, this list of conditions and the following disclaimer. 17 18 2. Redistributions in binary form must reproduce the above copyright 19 notice, this list of conditions and the following disclaimer in 20 the documentation and/or other materials provided with the 21 distribution. 22 23 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 24 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 25 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 26 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 27 COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 28 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 29 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 30 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 31 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 33 ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 POSSIBILITY OF SUCH DAMAGE. 35*/ 36 37:- module(persistency, 38 [ (persistent)/1, % +Declarations 39 current_persistent_predicate/1, % :PI 40 41 db_attach/2, % :File, +Options 42 db_detach/0, 43 db_attached/1, % :File 44 45 db_sync/1, % :What 46 db_sync_all/1, % +What 47 48 op(1150, fx, (persistent)) 49 ]). 50:- autoload(library(aggregate),[aggregate_all/3]). 51:- use_module(library(debug),[debug/3]). 52:- autoload(library(error), 53 [ instantiation_error/1, 54 must_be/2, 55 permission_error/3, 56 existence_error/2 57 ]). 58:- autoload(library(option),[option/3]). 59 60 61:- predicate_options(db_attach/2, 2, 62 [ sync(oneof([close,flush,none])) 63 ]).
138:- meta_predicate 139 db_attach(, ), 140 db_attached(), 141 db_sync(), 142 current_persistent_predicate(). 143:- module_transparent 144 db_detach/0. 145 146 147 /******************************* 148 * DB * 149 *******************************/ 150 151:- dynamic 152 db_file/5, % Module, File, Created, Modified, EndPos 153 db_stream/2, % Module, Stream 154 db_dirty/2, % Module, Deleted 155 db_option/2. % Module, Name(Value) 156 157:- volatile 158 db_stream/2. 159 160:- multifile 161 (persistent)/3, % Module, Generic, Term 162 prolog:generated_predicate/1. 163 164 165 /******************************* 166 * DECLARATIONS * 167 *******************************/
:- persistent
<callable>,
<callable>,
...
Each specification is a callable term, following the conventions of library(record), where each argument is of the form
name:type
Types are defined by library(error).
188persistent(Spec) :- 189 throw(error(context_error(nodirective, persistent(Spec)), _)). 190 191compile_persistent(Var, _, _) --> 192 { var(Var), 193 !, 194 instantiation_error(Var) 195 }. 196compile_persistent(M:Spec, _, LoadModule) --> 197 !, 198 compile_persistent(Spec, M, LoadModule). 199compile_persistent((A,B), Module, LoadModule) --> 200 !, 201 compile_persistent(A, Module, LoadModule), 202 compile_persistent(B, Module, LoadModule). 203compile_persistent(Term, Module, LoadModule) --> 204 { functor(Term, Name, Arity), % Validates Term as callable 205 functor(Generic, Name, Arity), 206 qualify(Module, LoadModule, Name/Arity, Dynamic) 207 }, 208 [ :- dynamic(Dynamic), 209 210 persistency:persistent(Module, Generic, Term) 211 ], 212 assert_clause(asserta, Term, Module, LoadModule), 213 assert_clause(assert, Term, Module, LoadModule), 214 retract_clause(Term, Module, LoadModule), 215 retractall_clause(Term, Module, LoadModule). 216 217assert_clause(Where, Term, Module, LoadModule) --> 218 { functor(Term, Name, Arity), 219 atomic_list_concat([Where,'_', Name], PredName), 220 length(Args, Arity), 221 Head =.. [PredName|Args], 222 Assert =.. [Name|Args], 223 type_checkers(Args, 1, Term, Check), 224 atom_concat(db_, Where, DBActionName), 225 DBAction =.. [DBActionName, Module:Assert], 226 qualify(Module, LoadModule, Head, QHead), 227 Clause = (QHead :- Check, persistency:DBAction) 228 }, 229 [ Clause ]. 230 231type_checkers([], _, _, true). 232type_checkers([A0|AL], I, Spec, Check) :- 233 arg(I, Spec, ArgSpec), 234 ( ArgSpec = _Name:Type, 235 nonvar(Type), 236 Type \== any 237 -> Check = (must_be(Type, A0),More) 238 ; More = Check 239 ), 240 I2 is I + 1, 241 type_checkers(AL, I2, Spec, More). 242 243retract_clause(Term, Module, LoadModule) --> 244 { functor(Term, Name, Arity), 245 atom_concat(retract_, Name, PredName), 246 length(Args, Arity), 247 Head =.. [PredName|Args], 248 Retract =.. [Name|Args], 249 qualify(Module, LoadModule, Head, QHead), 250 Clause = (QHead :- persistency:db_retract(Module:Retract)) 251 }, 252 [ Clause ]. 253 254retractall_clause(Term, Module, LoadModule) --> 255 { functor(Term, Name, Arity), 256 atom_concat(retractall_, Name, PredName), 257 length(Args, Arity), 258 Head =.. [PredName|Args], 259 Retract =.. [Name|Args], 260 qualify(Module, LoadModule, Head, QHead), 261 Clause = (QHead :- persistency:db_retractall(Module:Retract)) 262 }, 263 [ Clause ]. 264 265qualify(Module, Module, Head, Head) :- !. 266qualify(Module, _LoadModule, Head, Module:Head). 267 268 269:- multifile 270 system:term_expansion/2. 271 272systemterm_expansion((:- persistent(Spec)), Clauses) :- 273 prolog_load_context(module, Module), 274 phrase(compile_persistent(Spec, Module, Module), Clauses).
282current_persistent_predicate(M:PName/Arity) :- 283 persistency:persistent(M, Generic, _), 284 functor(Generic, Name, Arity), 285 ( Name = PName 286 ; atom_concat(assert_, Name, PName) 287 ; atom_concat(retract_, Name, PName) 288 ; atom_concat(retractall_, Name, PName) 289 ). 290 291prologgenerated_predicate(PI) :- 292 current_persistent_predicate(PI). 293 294 295 /******************************* 296 * ATTACH * 297 *******************************/
close (close journal after write), flush
(default, flush journal after write) or none
(handle as fully buffered stream).
If File is already attached this operation may change the sync
behaviour.
If the file ends in an incomplete term, e.g., because the disk was full or the process was killed while writing, a warning is printed and the file is truncated to just after the last complete term.
317db_attach(Module:File, Options) :- 318 db_set_options(Module, Options), 319 db_attach_file(Module, File). 320 321db_set_options(Module, Options) :- 322 option(sync(Sync), Options, flush), 323 must_be(oneof([close,flush,none]), Sync), 324 ( db_option(Module, sync(Sync)) 325 -> true 326 ; retractall(db_option(Module, _)), 327 assert(db_option(Module, sync(Sync))) 328 ). 329 330db_attach_file(Module, File) :- 331 db_file(Module, Old, _, _, _), % we already have a db 332 !, 333 ( Old == File 334 -> ( db_stream(Module, Stream) 335 -> sync(Module, Stream) 336 ; true 337 ) 338 ; permission_error(attach, db, File) 339 ). 340db_attach_file(Module, File) :- 341 db_load(Module, File), 342 !. 343db_attach_file(Module, File) :- 344 assert(db_file(Module, File, 0, 0, 0)). 345 346db_load(Module, File) :- 347 retractall(db_file(Module, _, _, _, _)), 348 debug(db, 'Loading database ~w', [File]), 349 catch(setup_call_cleanup( 350 open(File, read, In, [ encoding(utf8), newline(posix), 351 encoding_errors(error) 352 ]), 353 load_db_end(In, Module, File, Created, EndPos), 354 close(In)), 355 error(existence_error(source_sink, File), _), fail), 356 debug(db, 'Loaded ~w', [File]), 357 time_file(File, Modified), 358 assert(db_file(Module, File, Created, Modified, EndPos)). 359 360db_load_incremental(Module, File) :- 361 db_file(Module, File, Created, _, EndPos0), 362 setup_call_cleanup( 363 ( open(File, read, In, [ encoding(utf8), newline(posix), 364 encoding_errors(error) 365 ]), 366 read_action(In, created(Created0)), 367 set_stream_position(In, EndPos0) 368 ), 369 ( Created0 == Created, 370 debug(db, 'Incremental load from ~p', [EndPos0]), 371 load_db_end(In, Module, File, _Created, EndPos) 372 ), 373 close(In)), 374 debug(db, 'Updated ~w', [File]), 375 time_file(File, Modified), 376 retractall(db_file(Module, File, Created, _, _)), 377 assert(db_file(Module, File, Created, Modified, EndPos)). 378 379load_db_end(In, Module, File, Created, End) :- 380 read_db_action(In, File, T0, End0), 381 ( T0 = created(Created) 382 -> read_db_action(In, File, T1, End1) 383 ; T1 = T0, 384 End1 = End0, 385 Created = 0 386 ), 387 load_db(T1, In, Module, File, End1, End). 388 389load_db(end_of_file, _, _, _, End, End) :- !. 390load_db(assert(Term), In, Module, File, _End0, End) :- 391 persistent(Module, Term, _Types), 392 !, 393 assert(Module:), 394 read_db_action(In, File, T1, End1), 395 load_db(T1, In, Module, File, End1, End). 396load_db(asserta(Term), In, Module, File, _End0, End) :- 397 persistent(Module, Term, _Types), 398 !, 399 asserta(Module:), 400 read_db_action(In, File, T1, End1), 401 load_db(T1, In, Module, File, End1, End). 402load_db(retractall(Term, Count), In, Module, File, _End0, End) :- 403 persistent(Module, Term, _Types), 404 !, 405 retractall(Module:), 406 set_dirty(Module, Count), 407 read_db_action(In, File, T1, End1), 408 load_db(T1, In, Module, File, End1, End). 409load_db(retract(Term), In, Module, File, _End0, End) :- 410 persistent(Module, Term, _Types), 411 !, 412 ( retract(Module:) 413 -> set_dirty(Module, 1) 414 ; true 415 ), 416 read_db_action(In, File, T1, End1), 417 load_db(T1, In, Module, File, End1, End). 418load_db(Term, In, Module, File, _End0, End) :- 419 print_message(error, persistency(illegal_term(File, Term))), 420 read_db_action(In, File, T1, End1), 421 load_db(T1, In, Module, File, End1, End).
429read_db_action(In, File, Action, End) :-
430 stream_property(In, position(Start)),
431 catch(( read_action(In, Action),
432 stream_property(In, position(End))
433 ),
434 Error,
435 recover_db(Error, In, File, Start, Action, End)).Note that we do not repair a file from which not a single term could be read (Pos is 0). Such a file is most likely not a persistent database at all and we do not want to destroy it.
450recover_db(error(syntax_error(Culprit), _), In, File, Pos, Action, End) :- 451 at_end_of_stream(In), 452 stream_position_data(byte_count, Pos, Byte), 453 Byte > 0, 454 !, 455 print_message(warning, persistency(truncated_db(File, Pos, Culprit))), 456 truncate_db_file(File, Byte), 457 Action = end_of_file, 458 End = Pos. 459recover_db(Error, _In, _File, _Pos, _Action, _End) :- 460 throw(Error).
468truncate_db_file(File, Byte) :- 469 setup_call_cleanup( 470 open(File, update, Out, [type(binary), lock(write)]), 471 ( seek(Out, Byte, bof, _), 472 put_byte(Out, 0'\n), 473 set_end_of_stream(Out) 474 ), 475 close(Out)). 476 477db_clean(Module) :- 478 retractall(db_dirty(Module, _)), 479 ( persistent(Module, Term, _Types), 480 retractall(Module:), 481 fail 482 ; true 483 ).
489db_size(Module, Total) :- 490 aggregate_all(sum(Count), persistent_size(Module, Count), Total). 491 492persistent_size(Module, Count) :- 493 persistent(Module, Term, _Types), 494 predicate_property(Module:Term, number_of_clauses(Count)).
500db_attached(Module:File) :-
501 db_file(Module, File, _Created, _Modified, _EndPos).509:- public 510 db_assert/1, 511 db_asserta/1, 512 db_retractall/1, 513 db_retract/1. 514 515db_assert(Term) :- with_mutex('$persistency', db_assert_sync(Term)). 516db_asserta(Term) :- with_mutex('$persistency', db_asserta_sync(Term)). 517db_retract(Term) :- with_mutex('$persistency', db_retract_sync(Term)). 518db_retractall(Term) :- with_mutex('$persistency', db_retractall_sync(Term)). 519 520db_assert_sync(Module:Term) :- 521 assert(Module:), 522 persistent(Module, assert(Term)). 523 524db_asserta_sync(Module:Term) :- 525 asserta(Module:), 526 persistent(Module, asserta(Term)). 527 528persistent(Module, Action) :- 529 ( db_stream(Module, Stream) 530 -> true 531 ; db_file(Module, File, _Created, _Modified, _EndPos) 532 -> db_sync(Module, update), % Is this correct? 533 db_open_file(File, append, Stream), 534 assert(db_stream(Module, Stream)) 535 ; existence_error(db_file, Module) 536 ), 537 write_action(Stream, Action), 538 sync(Module, Stream).
created(Stamp) term. If we append to a file that does not end in a
layout character, e.g., because it was truncated in the layout that
follows the last term, we complete the line first. Without this the
new term is glued to the `.` of the last one.
All database streams use newline(posix). This keeps the file format
platform independent and, more importantly, makes the byte counts we
use to truncate a damaged file (see truncate_db_file/2) agree with
what the reader sees. On Windows a text stream deletes all carriage
returns, so a \r written as part of a newline is neither a byte we
can account for nor layout that separates two terms.
555db_open_file(File, Mode, Stream) :- 556 ( Mode == append, 557 exists_file(File), 558 \+ db_ends_with_layout(File) 559 -> Complete = true 560 ; Complete = false 561 ), 562 open(File, Mode, Stream, 563 [ close_on_abort(false), 564 encoding(utf8), 565 newline(posix), 566 lock(write) 567 ]), 568 ( size_file(File, 0) 569 -> get_time(Now), 570 write_action(Stream, created(Now)) 571 ; Complete == true 572 -> nl(Stream) 573 ; true 574 ). 575 576db_ends_with_layout(File) :- 577 size_file(File, Size), 578 Size > 0, 579 Last is Size-1, 580 setup_call_cleanup( 581 open(File, read, In, [type(binary)]), 582 ( seek(In, Last, bof, _), 583 get_byte(In, Byte) 584 ), 585 close(In)), 586 layout_byte(Byte). 587 588layout_byte(0'\n). 589layout_byte(0'\r). 590layout_byte(0' ). 591layout_byte(0'\t).
602db_detach :-
603 context_module(Module),
604 db_sync(Module:detach),
605 db_clean(Module).close, the journal
file is closed, making it easier to edit the file externally.
Using flush flushes the stream but does not close it. This
provides better performance. Using none, the stream is not
even flushed. This makes the journal sensitive to crashes, but
much faster.617sync(Module, Stream) :- 618 db_option(Module, sync(Sync)), 619 ( Sync == close 620 -> db_sync(Module, close) 621 ; Sync == flush 622 -> flush_output(Stream) 623 ; true 624 ). 625 626read_action(Stream, Action) :- 627 read_term(Stream, Action, [module(db)]). 628 629write_action(Stream, Action) :- 630 \+ \+ ( numbervars(Action, 0, _, [singletons(true)]), 631 format(Stream, '~W.~n', 632 [ Action, 633 [ quoted(true), 634 numbervars(true), 635 module(db) 636 ] 637 ]) 638 ).
646db_retractall_sync(Module:Term) :-
647 ( var(Term)
648 -> forall(persistent(Module, Term, _Types),
649 db_retractall(Module:Term))
650 ; State = count(0),
651 ( retract(Module:),
652 arg(1, State, C0),
653 C1 is C0+1,
654 nb_setarg(1, State, C1),
655 fail
656 ; arg(1, State, Count)
657 ),
658 ( Count > 0
659 -> set_dirty(Module, Count),
660 persistent(Module, retractall(Term, Count))
661 ; true
662 )
663 ).670db_retract_sync(Module:Term) :- 671 ( var(Term) 672 -> instantiation_error(Term) 673 ; retract(Module:), 674 set_dirty(Module, 1), 675 persistent(Module, retract(Term)) 676 ). 677 678 679set_dirty(_, 0) :- !. 680set_dirty(Module, Count) :- 681 ( retract(db_dirty(Module, C0)) 682 -> true 683 ; C0 = 0 684 ), 685 C1 is C0 + Count, 686 assert(db_dirty(Module, C1)).
reload, but use incremental loading if possible.
This allows for two processes to examine the same database
file, where one writes the database and the other periodycally
calls db_sync(update) to follow the modified data.gc(50).With unbound What, db_sync/1 reloads the database if it was modified on disk, gc it if it is dirty and close it if it is opened.
719db_sync(Module:What) :- 720 db_sync(Module, What). 721 722 723db_sync(Module, reload) :- 724 \+ db_stream(Module, _), % not open 725 db_file(Module, File, _Created, ModifiedWhenLoaded, _EndPos), 726 catch(time_file(File, Modified), _, fail), 727 Modified > ModifiedWhenLoaded, % Externally modified 728 !, 729 debug(db, 'Database ~w was externally modified; reloading', [File]), 730 !, 731 ( catch(db_load_incremental(Module, File), 732 E, 733 ( print_message(warning, E), fail )) 734 -> true 735 ; db_clean(Module), 736 db_load(Module, File) 737 ). 738db_sync(Module, gc) :- 739 !, 740 db_sync(Module, gc(50)). 741db_sync(Module, gc(When)) :- 742 ( When == always 743 -> true 744 ; db_dirty(Module, Dirty), 745 db_size(Module, Total), 746 ( Total > 0 747 -> Perc is (100*Dirty)/Total, 748 Perc > When 749 ; Dirty > 0 750 ) 751 ), 752 !, 753 db_sync(Module, close), 754 db_file(Module, File, _, Modified, _), 755 atom_concat(File, '.new', NewFile), 756 debug(db, 'Database ~w is dirty; cleaning', [File]), 757 get_time(Created), 758 catch(setup_call_cleanup( 759 db_open_file(NewFile, write, Out), 760 ( persistent(Module, Term, _Types), 761 call(Module:Term), 762 write_action(Out, assert(Term)), 763 fail 764 ; stream_property(Out, position(EndPos)) 765 ), 766 close(Out)), 767 Error, 768 ( catch(delete_file(NewFile),_,fail), 769 throw(Error))), 770 retractall(db_file(Module, File, _, Modified, _)), 771 rename_file(NewFile, File), 772 time_file(File, NewModified), 773 assert(db_file(Module, File, Created, NewModified, EndPos)). 774db_sync(Module, close) :- 775 retract(db_stream(Module, Stream)), 776 !, 777 db_file(Module, File, Created, _, _), 778 debug(db, 'Database ~w is open; closing', [File]), 779 stream_property(Stream, position(EndPos)), 780 close(Stream), 781 time_file(File, Modified), 782 retractall(db_file(Module, File, _, _, _)), 783 assert(db_file(Module, File, Created, Modified, EndPos)). 784db_sync(Module, Action) :- 785 Action == detach, 786 !, 787 ( retract(db_stream(Module, Stream)) 788 -> close(Stream) 789 ; true 790 ), 791 retractall(db_file(Module, _, _, _, _)), 792 retractall(db_dirty(Module, _)), 793 retractall(db_option(Module, _)). 794db_sync(_, nop) :- !. 795db_sync(_, _).
802db_sync_all(What) :- 803 must_be(oneof([reload,gc,gc(_),close]), What), 804 forall(db_file(Module, _, _, _, _), 805 db_sync(Module:What)). 806 807 808 /******************************* 809 * CLOSE * 810 *******************************/ 811 812close_dbs :- 813 forall(retract(db_stream(_Module, Stream)), 814 close(Stream)). 815 816:- at_halt(close_dbs). 817 818 819 /******************************* 820 * MESSAGES * 821 *******************************/ 822 823:- multifile 824 prolog:message//1. 825 826prologmessage(persistency(Message)) --> 827 message(Message). 828 829message(truncated_db(File, Pos, Culprit)) --> 830 { stream_position_data(line_count, Pos, Line) }, 831 [ 'Persistent database ~w is incomplete (~w).'-[File, Culprit], nl, 832 'Truncated the file to line ~d, the last complete term.'-[Line] 833 ]. 834message(illegal_term(File, Term)) --> 835 [ 'Persistent database ~w: ignored illegal term ~p'-[File, Term] ]
Provide persistent dynamic predicates
This module provides simple persistent storage for one or more dynamic predicates. A database is always associated with a module. A module that wishes to maintain a database must declare the terms that can be placed in the database using the directive persistent/1.
The persistent/1 expands each declaration into five predicates:
name(Arg, ...)assert_name(Arg, ...)asserta_name(Arg, ...)retract_name(Arg, ...)retractall_name(Arg, ...)As mentioned, a database can only be accessed from within a single module. This limitation is on purpose, forcing the user to provide a proper API for accessing the shared persistent data.
This module requires the same thread-synchronization as the normal Prolog database. This implies that if each individual assert or retract takes the database from one consistent state to the next, no additional locking is required. If more than one elementary database operation is required to get from one consistent state to the next, both updating and querying the database must be locked using with_mutex/2.
Below is a simple example, where adding a user does not need locking as it is a single assert, while modifying a user requires both a retract and assert and thus needs to be locked.
:- module(user_db, [ attach_user_db/1, % +File current_user_role/2, % ?User, ?Role add_user/2, % +User, +Role set_user_role/2 % +User, +Role ]). :- use_module(library(persistency)). :- persistent user_role(name:atom, role:oneof([user,administrator])). attach_user_db(File) :- db_attach(File, []). %% current_user_role(+Name, -Role) is semidet. current_user_role(Name, Role) :- with_mutex(user_db, user_role(Name, Role)). add_user(Name, Role) :- assert_user_role(Name, Role). set_user_role(Name, Role) :- user_role(Name, Role), !. set_user_role(Name, Role) :- with_mutex(user_db, ( retractall_user_role(Name, _), assert_user_role(Name, Role))).