\documentclass[conference]{IEEEtran}
\usepackage[utf8]{inputenc}

\usepackage{cite}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{textcomp}
\usepackage{xcolor}
\def\BibTeX{{\rm B\kern-.05em{\sc i\kern-.025em b}\kern-.08em
    T\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}


\usepackage{listings}
\usepackage{lstautogobble}
\usepackage{hyperref}

\input{lstisabelle}
\lstset{basicstyle=\footnotesize\ttfamily\slshape}
\lstset{captionpos=b}
\lstset{numberbychapter=false}
\lstset{autogobble=true}

\newcommand{\isai}{\lstinline[language=isabelle,basicstyle=\normalsize\ttfamily\slshape]}
\newcommand{\lsti}{\lstinline[language={},literate={}]}


% \input{lstpseudo}


% add a key to suppress counting of blank lines in listings
% source http://tex.stackexchange.com/questions/33999/
\makeatletter
\lst@Key{countblanklines}{true}[t]%
    {\lstKV@SetIf{#1}\lst@ifcountblanklines}

\lst@AddToHook{OnEmptyLine}{%
    \lst@ifnumberblanklines\else%
       \lst@ifcountblanklines\else%
         \advance\c@lstnumber-\@ne\relax%
       \fi%
    \fi}
\makeatother

% make lstinline work in math mode
\makeatletter
\renewcommand\lstinline[1][]{%
  \leavevmode
  \ifmmode\expandafter\hbox\fi\bgroup
    \def\lst@boxpos{b}%
    \lsthk@PreSet\lstset{flexiblecolumns,#1}%
    \lsthk@TextStyle
    \@ifnextchar\bgroup{\afterassignment\lst@InlineG \let\@let@token}%
                       \lstinline@}
\makeatother

\newcommand{\withlinenumbers}{%
  \lstset{numbers=left,numberstyle=\scriptsize,xleftmargin=2em,numberblanklines=false,countblanklines=false,escapechar=@}%
}


% \input{macros}

\newcommand\CC{C\nolinebreak[4]\hspace{-.05em}\raisebox{.4ex}{\relsize{-3}{\textbf{++}}}}

% % Include snippets
% \newcommand{\DefineSnippet}[2]{%
%    \expandafter\newcommand\csname snippet--#1\endcsname{%
%      \begin{quote}
%      \begin{isabelle}\footnotesize
%      #2
%      \end{isabelle}
%      \end{quote}}}
% \newcommand{\Snippet}[1]{\ifcsname snippet--#1\endcsname\csname snippet--#1\endcsname\else\PackageError{}{No snippet '#1' defined.}{}\fi}
% %\input{snippets.out}


\newcommand{\todo}[1]{\par\noindent\fbox{\parbox{\textwidth}{\color{darkgray}#1}}}

%
\begin{document}

\title{GRAT: a formally verified (UN)SAT proof checker\\{\footnotesize Proof Checker Proposal}}
% \titlerunning{Formalizing the Edmonds-Karp Algorithm}
% \subtitle{Proof Checker Proposal}

\author{\IEEEauthorblockN{Peter Lammich}
\IEEEauthorblockA{\textit{FMT Group, EEMCS department} \\
\textit{University of Twente}\\
Enschede, The Netherlands \\
p.lammich@utwente.nl 0000-0003-3576-0504}
}

\maketitle

\begin{abstract}
We propose the GRAT proof checker toolchain as a verified proof checker suitable for SAT competitions.
It accepts proofs in the DRAT format, and is verified down to a functional implementation in Standard ML.
On benchmarks drawn from recent SAT competitions, it's performance is similar to that of drat-trim.
\end{abstract}

% \begin{abstract}
% We present an efficient, formally verified checker for satisfiability and unsatisfiability certificates for 
% Boolean formulas in conjunctive normal form. 
% While satisfiability certificates are straightforward to check, the unsatisfiability checker utilizes a two phase approach: 
% Starting from a DRAT certificate, the unverified generator computes an enriched certificate,
% which is then checked against the original formula by the verified checker.
% 
% The actual implementation of the checker is mechanically verified with the Isabelle/HOL theorem prover, 
% against a specification of the semantics of the formula down to the integer sequence by which it is represented.
% 
% On a realistic benchmark suite drawn from the 2016 and 2017 SAT competitions, our approach is significantly faster 
% than the unverified standard tool {\sl drat-trim}. An optional multi-threaded mode of the generator further reduces the 
% runtime, in particular for big problems.
% \end{abstract}

\section{Introduction}

The GRAT toolchain accepts DRAT (ASCII and binary format) as input.
The result is formally verified with Isabelle/HOL, down to the integer sequence representing the formula.
The trusted code base of the verification is Isabelle/HOL's kernel and code generator,
compilation and running of the extracted Standard ML code with MLton, and a thin command line wrapper and formula parser
written in Standard ML.

Our tool chain follows a two step approach, with a highly optimized but unverified first step,
and a formally verified second step. As the first step only acts as certificate preprocessor,
it is not part of the trusted code base.

On a set of benchmarks drawn from the 2016 and 2017 SAT competitions, our full toolchain performed faster
than the unverified (then state-of-the-art) tool {\sl drat-trim}.
We have confirmed that our tool is still usable for modern SAT competitions, by testing it on benchmarks from the 2022 SAT competition.

A detailed description can be found in~\cite{La17_CADE,La17_SAT} and~\cite{La20}.
Here, we briefly summarize the main aspects, and report on the new set of benchmarks.

GRAT's webpage is \url{https://www21.in.tum.de/~lammich/grat/}, and the project is maintained as part of the IsaFOL repository \url{https://bitbucket.org/isafol/isafol/src/master/GRAT/}.

Download and build instructions are on the webpage.


\section{Proof Format}
Our toolchain supports the de-facto standard DRAT-format as input~\cite{WHH14}.

This is then processed by the unverified {\sl gratgen} tool, which produces a certificate enriched with unit
propagation information, in the GRAT-format. The GRAT certificate and the original formula
is then passed to the verified {\sl gratchk} tool, which either confirms unsatisfiability of the formula by printing
the status line {\sl s VERIFIED UNSAT}, or yields an error.

In the following, we sketch the GRAT-format.

Each clause is identified by a unique positive ID.
The clauses of the original formula implicitly get the IDs $1\ldots N$. The lemma IDs explicitly occur in the certificate.

For memory efficiency reasons, we store the certificate in two parts: The lemma file contains the lemmas, and is stored in DIMACS format.
During certificate checking, this part is entirely loaded into memory.
The proof file contains the hints and instructions for the certificate checker. It is not completely loaded into memory but only streamed during checking.

The proof file is a binary file, containing a sequence (stored in reverse order) of 32 bit signed integers in 2's complement little endian format.
The sequence is interpreted according to the following grammar:
\begin{lstlisting}[language={},columns={[c]fullflexible},literate={}]
  proof      ::= rat-counts item* conflict
  literal    ::= int32 != 0
  id         ::= int32 > 0
  count      ::= int32 > 0
  rat-counts ::= 6 (literal count)* 0
  item       ::= uprop | del | rup-lem | rat-lem
  uprop      ::= 1 id* 0
  del        ::= 2 id* 0
  rup-lem    ::= 3 id id* 0 id
  rat-lem    ::= 4 literal id id* 0 cand-prf* 0
  cand-prf   ::= id id* 0 id
  conflict   ::= 5 id
\end{lstlisting}

The checker maintains a \emph{clause map} that maps IDs to clauses, and a \emph{partial assignment} that maps variables to true, false, or undecided.
Partial assignments are extended to literals in the natural way.
Initially, the clause map contains the clauses of the original formula, and the partial assignment maps all variables to undecided.
Then, the checker iterates over the items of the proof, processing each item as follows:
\begin{itemize}
  \item {\tt rat-counts} This item contains a list of pairs of literals and the count how often they are used in RAT proofs.
      This map allows the checker to maintain lists of RAT candidates for the relevant literals, instead of gathering the
      possible RAT candidates by iterating over the whole clause database for each RAT proof, which is expensive.
      Literals that are not used in RAT proofs at all do not occur in the list. This item is the first item of the proof.
  \item {\tt uprop}
    For each listed clause ID, the corresponding clause is checked to be unit, and the unit literal is assigned to true.
    Here, a clause is unit if the unit literal is undecided, and all other literals are assigned to false.
  \item {\tt del} The specified IDs are removed from the clause map.
  \item {\tt rup-lem} The item specifies the ID for the new lemma, which is the next unprocessed lemma from the lemma file, a list of unit clause IDs, and a conflict clause ID.
      First, the literals of the lemma are assigned to false. The lemma must not be blocked, i.e.\ none of its literals may be already assigned to true\footnote{Blocked lemmas are useless for unsat proofs, such that there is no point to include them in the certificate.}.
        Note that assigning the literals of a clause $C$ to false is equivalent to adding the conjunct $\neg C$ to the formula.
      Second, the unit clauses are checked and the corresponding unit literals are assigned to true.
      Third, it is checked that the conflict clause ID actually identifies a conflict clause, i.e.\ that all its literals are assigned to false.
      Finally, the lemma is added to the clause-map and the assignment is rolled back to the state before checking of the item started.
  \item {\tt rat-lemma} The item specifies a pivot literal $l$, an ID for the lemma, an initial list of unit clause IDs, and a list of
      candidate proofs.
      First, as for \lsti{rup-lemma}, the literals of the lemma are assigned to false and the initial unit propagations are performed.
      Second, it is checked that the provided RAT candidates are exhaustive, and then the corresponding \lsti{cand-prf} items are processed:
      A \lsti{cand-prf} item consists of the ID of the candidate clause $D$, a list of unit clause IDs, and a conflict clause ID.
      To check a candidate proof, the literals of $D\setminus\{\neg l\}$ are assigned to false, the listed unit propagations are performed, and the conflict clause is
      checked to be actually conflict. Afterwards, the assignment is rolled back to the state before checking the candidate proof.
      Third, when all candidate proofs have been checked, the lemma is added to the clause map and the assignment is rolled back.

      To simplify certificate generation in backward mode, we allow candidate proofs referring to arbitrary, even invalid, clause IDs. Those proofs must be ignored by the checker.
  \item {\tt conflict} This is the last item of the certificate. It specifies the ID of the conflict clause found by unit propagation after adding the last
    lemma of the certificate (\emph{root conflict}). It is checked that the ID actually refers to a conflict clause.
\end{itemize}


\section{Evaluation}

\subsection{Usage Example}
We give a simple example on how to use our toolchain:

To verify that a formula stored in the DIMACS file {\tt unsat.cnf} is unsatisfiable, proceed as follows:

{\footnotesize
\begin{verbatim}
# Create a (binary) drat-file
> kissat -q unsat.cnf unsat.drat
s UNSATISFIABLE
# Process into proof (gratp) and lemmas (gratl) file
> gratgen unsat.cnf unsat.drat \
    -o unsat.gratp -l unsat.gratl -b
s VERIFIED
# Check against original formula
> gratchk unsat unsat.{cnf,gratl,gratp}
s VERIFIED UNSAT
\end{verbatim}
}

To verify that a formula stored in the DIMACS file {\tt sat.cnf} is satisfiable, proceed as follows:

{\footnotesize
\begin{verbatim}
# Produce variable assignment,
# as 0-terminated list of literals
> kissat -q sat.cnf | grep "^v" \
    | sed -re 's/^v//g' > sat.vars
# Check against original formula
> gratchk sat sat.{cnf,vars}
s VERIFIED SAT
\end{verbatim}
}

\subsection{MLtons Memory Manager}
When running gratchk on machines with a lot of memory, we ran into two problems with MLtons default memory manager:
First it will take half of the machine's memory before even starting to garbage collect. And, second, when it garbage collects, it will try to keep allocated 8 times the live memory size. Both behaviours are problematic: small problems will consume huge amounts of memory, making it impossible to verify many small problems in parallel on the same machine. Also, most of the memory that gratchk consumes is the storage for the formula and lemmas. Once the checking starts, only little additional memory is needed. However, MLtons memory manager will try to allocate 8 times the live size, which includes the (potentially large) formula and lemmas. In practice, this led to gratchk processes being killed by the out-of-memory killer.

While there is no ideal solution currently supported by MLton, we decided to apply a simple heuristic and limit the memory available to gratchk to 10 times the formula and lemma file size, and a minimum of 1GiB. In practice, this can be achieved by system tools, or by a runtime option to MLton, e.g.:

{\footnotesize
\begin{verbatim}
> gratchk @MLton max-heap 2G -- \
    unsat unsat.{cnf,gratl,gratp}
\end{verbatim}
}


\subsection{Theoretical Complexity}
Our toolchain has polynomial complexity in the size of the input (drat) certificate and formula.
While we have not estimated the precise complexities, we give a rough argument that the complexity is polynomial.

The first phase, {\sl gratgen}, iterates over each clause in the certificate, and puts it into a two-watched-literals (twl) data structure. This clearly takes polynomial time. It then iterates backwards over the clauses.
For each clause, the (inverted) literals are added as units, and then unit-propagation is performed.
This also takes polynomial time. In case of a RAT clause, further clauses are gathered from the available clauses,
and for each of those, another unit propagation is done (again, polynomial unit propagation for linearly many clauses).
After checking each clause, the twl data structure is reverted to the state before that clause (which also takes polynomial time).

The second phase, {\sl gratchk}, repeats the actions from the first phase, but iterating in a forwards fashion, and using extra information for unit propagation. Thus, it is also clearly polynomial.


\subsection{Empirical Evaluation}
  We have extensively benchmarked our toolchain in~\cite{La20}, where we also compared it
  against the then-current versions of {\sl drat-trim} and LRAT~\cite{HHKW17}.

  Our tool has not significantly changed since then, and we refer the reader to~\cite{La20} for those results.

  To check if our tool is still usable, we have run it on problems from the 2022 SAT competition's main track.
  We considered the winning solver Kissat\_MAB\-HyWalk, and the highest ranked non-Kissat based solver SeqFROST-ERE-All. We ran the solvers on all unsatisfiable problems they could solve in the competition to regenerate the certificates, and then used GRAT to verify the results.
  We benchmarked two configurations for gratgen: single-threaded and 8 parallel threads.
  Previous experiments have shown that more than 8 threads do not bring significant speedup.

  \subsubsection{Verified Problems}
  First of all, we could verify all 146 problems for Kissat and all 138 problems for SeqFROST.
  The single-threaded gratgen timed out on one Kissat problem, though.

  \subsubsection{Solving vs.\ Verification time}
  We compare the solving time with the verification time. Let $t_s$ be the solving time, and $t_v$ be the verification time, we compute, for each problem the ratio $r=t_v/t_s$, and then count for what percentage of the problems this ratio is less than $.5$, $1$, $2$, and $4$. This is a sensible measure, as we expect the verification time to be related to the difficulty of the problem, and thus the solving time. Also, it estimates the extra time required to get a verified result.
  The result is displayed in the following table:

  {\vspace{1em}
  \begin{tabular}{|l|c|c|c|c|c|}\hline
                & $<.5$ & $<1$ & $<2$ & $<4$ & \#problems\\
    \hline\hline
    Kissat-j8   & 70.5 & 85.6 & 93.8 & 97.3  & 146 \\
    SeqFROST-j8 & 76.8 & 89.1 & 96.4 & 99.3  & 138 \\
    Kissat-j1   & 26.9 & 60.0 & 87.6 & 93.8  & 145 \\
    SeqFROST-j1 & 24.6 & 50.7 & 81.9 & 97.1  & 138 \\
    \hline
  \end{tabular}
  \vspace{1em}
  }

  That is, with 8 threads, we can verify more than 80\% of the problems when allowing the same time for verification as for solving. In single-threaded mode, it's still more than half of the problems. And more than 90\% of the problems will be solved when allowing a factor of 2 (8 threads) or 4 (1 thread), respectively.

  \subsubsection{Drat Certificate vs Grat Certificate Size}
  Next, we compare the size of the drat certificate produced by the SAT solver to the size of the enriched (grat) certificate produced by the first phase of our tool. This is of concern as the certificates have to be stored on disk, and thus, should not be excessively big. As for the time, we determine the ratio grat-size over drat-size, and count the percentage of problems below certain ratios.

  {\vspace{1em}
  \begin{tabular}{|l|c|c|c|c|c|}\hline
                & $<.5$ & $<1$ & $<2$ & $<4$ & \#problems\\
    \hline\hline
    Kissat      & 46.6 & 54.1 & 84.9 & 97.9 & 146\\
    SeqFROST    & 50.0 & 54.3 & 81.9 & 97.8 & 138\\
    \hline
  \end{tabular}
  \vspace{1em}
  }

  We observe that the generated grat certificate is smaller than the original drat certificate in more than half of the cases, and rarely exceeds factor 4.
  This is due to the trimming heuristics in gratgen, which, similar to drat-trim, tries hard to eliminate as many useless lemmas as possible.
  In many cases, this elimination removes more than the extra unit-propagation information that is added by gratgen.

%   \subsubsection{Speedup for Parallel Verification Mode}
%   To get an impression of how much effect multithreading in gratgen has, we compared the runs with 1 thread and the runs with 8 threads. For the 128 Kissat problems for that we managed to measure usable timings in both 1 and 8 thread mode, the overall verification time was 512739s in single-threaded mode, vs.~ 85408s with 8 threads, which is a speed-up of roughly factor 6. For SeqFROST, we had usable measurements for 113 problems, and needed 295155s in single-threaded mode, vs.~ 68235s with 8 threads, which is a speed-up of roughly 4.3.


\section{Formal Verification}
The crucial part of our toolchain is the {\sl gratchk} tool, which takes as input the original formula and a
certificate in GRAT format, and then verifies that the formula is actually unsatisfiable.
It also supports a mode for verifying satisfiable formulas, which takes a list of true literals as proof.

The {\sl gratchk} tool is written in Standard ML, and compiled using the MLton compiler.
The top-level is an unverified command line interface, which interprets the commands, and parses the specified files
into an array of integers. The array contains a representation of the formula, followed by a representation of the proof.
It then calls the core functions \isai{verify_sat_impl} and \isai{verify_unsat_split_impl}, which are
exported from an Isabelle formalization using Isabelle's code generator.

\begin{lstlisting}{language=SML}
  val verify_sat_impl
    : int array -> nat -> unit -> (_, _) sum
  val verify_unsat_split_impl
    : int array -> ('a -> int * 'a)
      -> nat -> nat -> nat * 'a -> unit -> (_, _) sum
\end{lstlisting}

For these functions, we have proved the following lemmas in Isabelle:

\begin{lstlisting}
theorem verify_sat_impl_correct: "
  <DBi |->$_a$ DB>
    verify_sat_impl DBi F_end
  <\<lambda>result. DBi |->$_a$ DB
    * \<up>(\<not>isl result ==> verify_sat_spec DB F_end)>"

theorem verify_unsat_impl_correct: "
  <DBi |->$_a$ DB>
    verify_unsat_split_impl DBi prfn F_end it prf
  <\<lambda>result. DBi |->$_a$ DB
    * \<up>(\<not>isl result ==> verify_unsat_spec DB F_end)>"
\end{lstlisting}
The preconditions of these Hoare triples state that the argument \isai$DBi$ points to an array
holding the elements \isai$DB$. This array is not changed by the functions (it occurs unchanged in the postcondition),
and these Hoare-triples imply termination of the program, as well as that it does not change any memory apart from what it
allocates itself.

The original formula is stored in \isai{DB[1..<F_end]}. (\isai$DB[0]$ is used as a guard by our implementation).
The result of the functions are from an exception monad, represented by a sum type.
The second parts of the postconditions state that, if no exception is raised, the formula stored at \isai{DB[1..<F_end]} is satisfiable or unsatisfiable respectively. In case of the unsat proof,
the other parameters \isai{prfn, it, prf} are used to represent the proof, but they have no influence on the statement
of this lemma: regardless of their values, an accepted formula is always unsat (If we pass nonsense, however, we will likely
get an exception).

To express when a formula is (un)sat, we have two (proved equivalent) specifications.
The first version relies on a function \isai{F_\<alpha>} that maps lists of integers to our internal
representation of SAT formulas, and the predicate \isai{sat} that specifies if a formula is satisfiable:

\begin{lstlisting}
definition "verify_sat_spec DB F_end ==
    1 <= F_end \<and> F_end <= length DB
  \<and> (let lst = tl (take F_end DB) in
       F_invar lst \<and> sat (F_\<alpha> lst))"

definition "verify_unsat_spec DB F_end ==
    1 <= F_end \<and> F_end <= length DB
  \<and> (let lst = tl (take F_end DB) in
       F_invar lst \<and> \<not>sat (F_\<alpha> lst))"
\end{lstlisting}

These specifications state that \isai{F_end} is in range, and that \isai{DB[1..<F_end]} (in Isabelle: \isai{tl (take F_end DB)}) is a valid (\isai{F_invar}) representation of a satisfiable or unsatisfiable, respectively, formula.

To increase the trust in these specifications, we prove them equivalent to a version that only relies on basic list operations:
First, we use the function \isai{tokenize :: int list => int list list}, which splits a list into its zero-terminated components.
To sanity-check this function, we prove that, for a list that ends with a zero (i.e., contains no open clause at the end), its result is the unique inverse of concatenation:
\begin{lstlisting}
definition "concat0 ll = concat (map (\<lambda>l . l@[0]) ll)"
lemma unique_tokenization:
  assumes "l~=[] ==> last l = 0"
  shows "\<exists>_1ls. (0\<notin>\<Union>set (map set ls) \<and> concat0 ls = l)"
    and "tokenize l = (THE ls.
          0\<notin>\<Union>set (map set ls) \<and> concat0 ls = l)"
\end{lstlisting}
where \isai{THE} is the definite description operator.

Next, we define an assignment from integers to Booleans to be consistent iff a negative value is mapped
to the opposite of its absolute value:
\begin{lstlisting}
definition assn_consistent :: "(int => bool) => bool"
  where "assn_consistent \<sigma>
    = (\<forall>x. x\<noteq>0 ==> \<not> \<sigma> (-x) = \<sigma> x)"
\end{lstlisting}
Finally, we characterize an (un)satisfiable input by the (non)existence of a consistent assignment that assigns at least one literal of each clause to true. Thus, we prove the following alternative characterizations of our specifications:
\begin{lstlisting}
lemma "verify_sat_spec DB F_end = (
  1<=F_end \<and> F_end <= length DB \<and> (
  let lst = tl (take F_end DB) in
    (lst~=[] ==> last lst = 0)
  \<and> (\<exists>\<sigma>. assn_consistent \<sigma>
        \<and> (\<forall>C\<in>set (tokenize 0 lst). \<exists>l\<in>set C. \<sigma> l))))"

lemma "verify_unsat_spec DB F_end = (
  1 < F_end \<and> F_end <= length DB \<and> (
  let lst = tl (take F_end DB) in
      last lst = 0
  \<and> (\<nexists>\<sigma>. assn_consistent \<sigma>
        \<and> (\<forall>C\<in>set (tokenize 0 lst). \<exists>l\<in>set C. \<sigma> l))))"
\end{lstlisting}
In the case of unsatisfiability, the bounds have been adjusted to exclude the empty formula, which is trivially satisfiable.


\subsection{Trusted Code Base}
Our approach relies on the correctness of the following components
\begin{itemize}
  \item Isabelle/HOL's inference kernel.
  \item Isabelle/HOL's code generator to Standard ML.
  \item The Imperative/HOL extension of the code generator.
  \item The correct formalization of what a Hoare-triple means.
  \item The correct specification of the correctness properties.
  \item The command line interface and DIMACs file parser.
  \item The correctness of the ML compiler and execution environment.
\end{itemize}

Where possible, we have tried to keep these trusted components as simple as possible.
For example, we have proved two equivalent forms of the correctness specification,
and limited the unverified parser to parse the DIMACs file into an array of integers.
The interpretation of these integers as list of clauses is done inside Isabelle.


\bibliographystyle{abbrv}
\bibliography{root}



\end{document}

