/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 *   SPDX-License-Identifier: Apache-2.0
 */

package org.apache.jena.tdb2.sys;

import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assumptions.assumeFalse;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.atomic.AtomicInteger;

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import org.apache.commons.io.FileUtils;
import org.apache.jena.atlas.lib.FileOps;
import org.apache.jena.base.Sys;
import org.apache.jena.dboe.base.file.Location;
import org.apache.jena.dboe.sys.IO_DB;
import org.apache.jena.dboe.transaction.txn.*;
import org.apache.jena.graph.Graph;
import org.apache.jena.graph.Triple;
import org.apache.jena.sparql.core.DatasetGraph;
import org.apache.jena.sparql.core.Quad;
import org.apache.jena.sparql.sse.SSE;
import org.apache.jena.system.Txn;
import org.apache.jena.tdb2.ConfigTest;
import org.apache.jena.tdb2.DatabaseMgr;
import org.apache.jena.tdb2.TDBException;
import org.apache.jena.tdb2.store.DatasetGraphSwitchable;
import org.apache.jena.tdb2.store.DatasetGraphTDB;

public class TestDatabaseCompact
{
    private Location dir = null;

    static Quad quad1 = SSE.parseQuad("(_ <s> <p> 1)");
    static Quad quad2 = SSE.parseQuad("(_ _:a <p> 2)");
    static Triple triple1 = quad1.asTriple();
    static Triple triple2 = quad2.asTriple();
    static Triple triple3 = SSE.parseTriple("(<s> <q> 3)");

    @BeforeEach
    public void before() {
        String DIR = ConfigTest.getCleanDir();
        FileOps.ensureDir(DIR);
        FileOps.clearAll(DIR);
        dir = Location.create(DIR);
    }

    @AfterEach
    public void after() {
        TDBInternal.reset();
        FileUtils.deleteQuietly(IO_DB.asFile(dir));
    }

    @Test public void compact_dsg_1() {
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
        DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
        DatasetGraph dsg1 = dsgs.get();
        Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();

        Txn.executeWrite(dsg, ()-> {
            dsg.add(quad2);
            dsg.add(quad1);
        });
        DatabaseMgr.compact(dsg, false);

        assertFalse(StoreConnection.isSetup(loc1));

        DatasetGraph dsg2 = dsgs.get();
        Location loc2 = ((DatasetGraphTDB)dsg2).getLocation();

        assertNotEquals(dsg1, dsg2);
        assertNotEquals(loc1, loc2);

        Txn.executeRead(dsg, ()-> {
            assertTrue(dsg.contains(quad2));
            assertTrue(dsg.contains(quad1));
        });

        // dsg1 was closed and expelled. We must carefully reopen its storage only.
        DatasetGraph dsgOld = StoreConnection.connectCreate(loc1).getDatasetGraph();

        Txn.executeWrite(dsgOld, ()->dsgOld.delete(quad2));
        Txn.executeRead(dsg,     ()->assertTrue(dsg.contains(quad2)) );
        Txn.executeRead(dsg2,    ()->assertTrue(dsg2.contains(quad2)) );
    }

    @Test public void compact_graph_2() {
        // graphs across compaction.
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
        Graph g = dsg.getDefaultGraph();

        DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
        DatasetGraph dsg1 = dsgs.get();
        Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();

        Txn.executeWrite(dsg, ()-> {
            dsg.add(quad2);
            dsg.add(quad1);
        });
        DatabaseMgr.compact(dsg, false);
        Txn.executeRead(dsg, ()-> {
            assertEquals(2, g.size());
            assertTrue(g.contains(triple2));
        });

        // Check is not attached to the old graph.
        DatasetGraph dsgOld = StoreConnection.connectCreate(loc1).getDatasetGraph();

        Txn.executeWrite(dsgOld, ()->dsgOld.getDefaultGraph().delete(triple2));
        Txn.executeRead(dsg,     ()->assertTrue(g.contains(triple2)) );

        Txn.executeWrite(dsg,    ()->g.add(triple3));
        Txn.executeRead(dsgOld,  ()->assertFalse(dsgOld.getDefaultGraph().contains(triple3)) );
    }

    @Test public void compact_prefixes_3() {
        // 2020-04:
        // Case 1:
        // This test fails with an NPE with Java11+ on ASF Jenkins.
        // java.lang.NullPointerException
        //         at java.base/java.nio.file.Files.provider(Files.java:105)
        //         at java.base/java.nio.file.Files.isDirectory(Files.java:2308)
        //         at org.apache.jena.tdb2.sys.IOX.asLocation(IOX.java:151)
        //         at org.apache.jena.tdb2.sys.DatabaseOps.compact(DatabaseOps.java:176)
        //         at org.apache.jena.tdb2.DatabaseMgr.compact(DatabaseMgr.java:62)
        //         at org.apache.jena.tdb2.sys.TestDatabaseOps.compact_prefixes_3_test(TestDatabaseOps.java:157)
        //         at org.apache.jena.tdb2.sys.TestDatabaseOps.compact_prefixes_3(TestDatabaseOps.java:135)
        // The NPE is from java.nio.file.Files.provider.
        // It seems to be impossible for the data from TDB to be null intermittently.
        // It does not fail anywhere else except at ASF and it does not always fail.
        // This seems to be on specific, and maybe just on, an ASF-Jenkins build node.

        // 2021-08:
        // Case 2:
        // Another "can't happen" situation on ASF Jenkins only.
        // This is consistent with case 1 - it just shows up differently now.
        // The database got created - so its directory must exist unless something external interfered.
        // No location: (/home/jenkins/workspace/Jena/Jena_Development_Deploy/jena-db/jena-tdb2/target/tdb-testing/DB,Data)
        //
        // Stacktrace
        //
        // org.apache.jena.tdb2.TDBException: No location: (/home/jenkins/workspace/Jena/Jena_Development_Deploy/jena-db/jena-tdb2/target/tdb-testing/DB,Data)
        //     at org.apache.jena.tdb2.sys.TestDatabaseOps.compact_prefixes_3_test(TestDatabaseOps.java:175)
        //     at org.apache.jena.tdb2.sys.TestDatabaseOps.compact_prefixes_3(TestDatabaseOps.java:145)

        try {
            compact_prefixes_3_test();
        } catch (NullPointerException ex) {
            ex.printStackTrace();
            //            StackTraceElement[] x = ex.getStackTrace();
            //            if ( x.length >= 0 && "java.nio.file.Files".equals(x[0].getClassName()) )
            //               return ;
            throw ex;
        }
        catch (TDBException ex) {
            if ( ex.getMessage().startsWith("No location:")) {
                System.err.println("'No location:' for known location. Msg-> " + ex.getMessage());
                return ;
            }
            throw ex;
        }
    }

    private void compact_prefixes_3_test() {
        // prefixes across compaction.
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
        Graph g = dsg.getDefaultGraph();
        Txn.executeWrite(dsg, ()-> g.getPrefixMapping().setNsPrefix("ex", "http://example/") );

        Txn.executeRead(dsg, ()-> {
            assertEquals("ex", g.getPrefixMapping().getNsURIPrefix("http://example/"));
            assertEquals("http://example/", g.getPrefixMapping().getNsPrefixURI("ex"));
        });

        DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
        assertNotNull(dsgs.getLocation(), ()->"DatasetGraphSwitchable created");
        DatasetGraph dsg1 = dsgs.get();
        Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();

        // Before
        int x1 = Txn.calculateRead(dsg, ()->dsg.prefixes().size());
        assertTrue(x1 > 0, ()->"Prefxies count");

        DatabaseMgr.compact(dsgs, false); // HERE

        // After
        int x2 = Txn.calculateRead(dsg, ()->dsg.prefixes().size());

        assertEquals(x1, x2, ()->"Before and after prefix count");

        Graph g2 = dsgs.getDefaultGraph();

        Txn.executeRead(dsgs, ()-> {
            assertEquals("ex", g2.getPrefixMapping().getNsURIPrefix("http://example/"));
            assertEquals("http://example/", g2.getPrefixMapping().getNsPrefixURI("ex"));
        });

        // Check is not attached to the old graph.
        DatasetGraph dsgOld = StoreConnection.connectCreate(loc1).getDatasetGraph();

        Txn.executeWrite(dsgOld, ()->dsgOld.getDefaultGraph().getPrefixMapping().removeNsPrefix("ex"));
        Txn.executeRead(dsg,     ()->assertEquals("http://example/", g.getPrefixMapping().getNsPrefixURI("ex")));

        Txn.executeWrite(dsg,    ()->g.getPrefixMapping().setNsPrefix("ex2", "http://exampl2/") );
        Txn.executeRead(dsgOld,  ()->assertNull(dsgOld.getDefaultGraph().getPrefixMapping().getNsPrefixURI("ex")));
    }

    @Test public void compact_delete() {
        assumeFalse(Sys.isWindows);
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
        DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
        DatasetGraph dsg1 = dsgs.get();
        Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();

        Txn.executeWrite(dsg, ()-> {
            dsg.add(quad2);
            dsg.add(quad1);
        });
        DatabaseMgr.compact(dsg, true);

        // Memory mapped files do not go away on MS Windows until the JVM exits.
        // This is a long standing JDK issue.
        // https://bugs.openjdk.java.net/browse/JDK-4715154
        if ( ! Sys.isWindows )
            assertFalse(IO_DB.asFile(loc1).exists());

        DatasetGraph dsg2 = dsgs.get();
        Location loc2 = ((DatasetGraphTDB)dsg2).getLocation();

        assertNotEquals(dsg1, dsg2);
        assertNotEquals(loc1, loc2);

        Txn.executeRead(dsg, ()-> {
            assertTrue(dsg.contains(quad2));
            assertTrue(dsg.contains(quad1));
        });

        Txn.executeRead(dsg,  ()->assertTrue(dsg.contains(quad2)) );
        Txn.executeRead(dsg2, ()->assertTrue(dsg2.contains(quad2)) );
    }

    @Test public void compact_external_components() {
        assumeFalse(Sys.isWindows);
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);

        TransactionCoordinator txnCoord = TDBInternal.getTransactionCoordinator(dsg);
        assertTrue(txnCoord.listExternals().isEmpty());

        ComponentId id = ComponentId.allocLocal(0xF00F);
        AtomicInteger counter = new AtomicInteger(0);

        TransactionalComponent txnExternal = new TransactionalComponentExternal<Void>(id) {
            @Override
            protected void _commit(TxnId txnId, Void state) { counter.getAndIncrement(); }
        };

        // Add external.
        txnCoord.modifyConfig(()->txnCoord.addExternal(txnExternal));

        { // Test internal state
            assertFalse(txnCoord.listExternals().isEmpty());
            assertEquals(1, txnCoord.listExternals().size());
        }

        assertEquals(0, counter.get());
        Txn.executeWrite(dsg, ()-> {
            dsg.add(quad2);
            dsg.add(quad1);
        });
        assertEquals(1, counter.get());

        DatabaseMgr.compact(dsg, true);

        { // Test internal state
            TransactionCoordinator txnCoord2 = TDBInternal.getTransactionCoordinator(dsg);
            assertNotSame(txnCoord, txnCoord2);
            assertEquals(1, txnCoord2.listExternals().size());
            assertSame(txnExternal, txnCoord2.listExternals().get(0));
        }

        Txn.executeWrite(dsg, ()->{});
        assertEquals(2, counter.get());
    }

    @Test public void compact_recovery_1() throws IOException {
        DatasetGraph dsg = DatabaseMgr.connectDatasetGraph(dir);
        Txn.executeWrite(dsg, ()->dsg.add(quad1));

        DatasetGraphSwitchable dsgs = (DatasetGraphSwitchable)dsg;
        DatasetGraph dsg1 = dsgs.get();
        Location loc1 = ((DatasetGraphTDB)dsg1).getLocation();

        // Mock some temporary files.
        Path pathTmp1 = dsgs.getContainerPath().resolve("Data-9999-tmp");
        Files.createDirectory(pathTmp1);
        Path pathTmp2 = dsgs.getContainerPath().resolve("Data-9998-tmp");
        Files.createDirectory(pathTmp2);
        Path pathFile2 = pathTmp2.resolve("data");
        Files.writeString(pathFile2, "Hello world");

        assertTrue(Files.exists(pathTmp1));
        assertTrue(Files.exists(pathFile2));
        assertTrue(Files.exists(pathTmp2));
        // Drop database
        TDBInternal.expel(dsg);

        // Reconnect
        DatasetGraph dsg2 = DatabaseMgr.connectDatasetGraph(dir);
        Txn.executeRead(dsg2, ()->assertTrue(dsg2.contains(quad1)));

        // These should no longer exist.
        assertFalse(Files.exists(pathTmp1));
        assertFalse(Files.exists(pathFile2));
        assertFalse(Files.exists(pathTmp2));
    }
}
