Source code

001/*
002 * Licensed to DuraSpace under one or more contributor license agreements.
003 * See the NOTICE file distributed with this work for additional information
004 * regarding copyright ownership.
005 *
006 * DuraSpace licenses this file to you under the Apache License,
007 * Version 2.0 (the "License"); you may not use this file except in
008 * compliance with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.fcrepo.integration.http.api;
019
020
021import com.fasterxml.jackson.databind.ObjectMapper;
022import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
023import edu.wisc.library.ocfl.api.OcflOption;
024import edu.wisc.library.ocfl.api.OcflRepository;
025import edu.wisc.library.ocfl.api.model.ObjectVersionId;
026import edu.wisc.library.ocfl.api.model.VersionInfo;
027import edu.wisc.library.ocfl.api.model.VersionNum;
028import org.apache.http.HttpStatus;
029import org.apache.http.client.methods.HttpGet;
030import org.apache.jena.graph.Node;
031import org.apache.jena.sparql.core.Quad;
032import org.fcrepo.config.FedoraPropsConfig;
033import org.fcrepo.http.commons.test.util.CloseableDataset;
034import org.fcrepo.kernel.api.FedoraTypes;
035import org.fcrepo.kernel.api.ReadOnlyTransaction;
036import org.fcrepo.kernel.api.Transaction;
037import org.fcrepo.kernel.api.identifiers.FedoraId;
038import org.fcrepo.kernel.impl.TransactionManagerImpl;
039import org.fcrepo.persistence.ocfl.RepositoryInitializer;
040import org.fcrepo.persistence.ocfl.api.FedoraOcflMappingNotFoundException;
041import org.fcrepo.persistence.ocfl.api.FedoraToOcflObjectIndex;
042import org.fcrepo.persistence.ocfl.impl.ReindexService;
043import org.fcrepo.storage.ocfl.ResourceHeaders;
044import org.junit.Assert;
045import org.junit.Before;
046import org.junit.Test;
047import org.slf4j.Logger;
048import org.slf4j.LoggerFactory;
049import org.springframework.test.context.TestExecutionListeners;
050
051import java.io.IOException;
052import java.io.UncheckedIOException;
053import java.net.URLDecoder;
054import java.nio.charset.StandardCharsets;
055import java.nio.file.Files;
056import java.nio.file.Paths;
057import java.time.Duration;
058import java.time.ZoneOffset;
059import java.util.Collections;
060import java.util.List;
061import java.util.concurrent.TimeUnit;
062import java.util.stream.Collectors;
063import java.util.stream.StreamSupport;
064
065import static java.text.MessageFormat.format;
066import static java.util.Arrays.asList;
067import static javax.ws.rs.core.Response.Status.GONE;
068import static javax.ws.rs.core.Response.Status.OK;
069import static org.apache.jena.graph.Node.ANY;
070import static org.apache.jena.graph.NodeFactory.createURI;
071import static org.fcrepo.kernel.api.FedoraTypes.FCR_METADATA;
072import static org.fcrepo.kernel.api.FedoraTypes.FCR_VERSIONS;
073import static org.fcrepo.kernel.api.RdfLexicon.CONTAINS;
074import static org.junit.Assert.assertEquals;
075import static org.junit.Assert.assertFalse;
076import static org.junit.Assert.fail;
077import static org.springframework.test.util.AssertionErrors.assertTrue;
078
079/**
080 * @author awooods
081 * @since 2020-03-04
082 */
083@TestExecutionListeners(listeners = { TestIsolationExecutionListener.class },
084        mergeMode = TestExecutionListeners.MergeMode.MERGE_WITH_DEFAULTS)
085public class RebuildIT extends AbstractResourceIT {
086
087    private static final Logger LOGGER = LoggerFactory.getLogger(RebuildIT.class);
088
089    private OcflRepository ocflRepository;
090    private RepositoryInitializer initializer;
091    private ReindexService reindexService;
092    private ObjectMapper objectMapper;
093    private FedoraPropsConfig fedoraPropsConfig;
094    private FedoraToOcflObjectIndex index;
095    private Transaction readOnlyTx;
096    private TransactionManagerImpl txManager;
097
098    private void setBeans() {
099        ocflRepository = getBean(OcflRepository.class);
100        initializer = getBean(RepositoryInitializer.class);
101        reindexService = getBean(ReindexService.class);
102        objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
103        fedoraPropsConfig = getBean(FedoraPropsConfig.class);
104        index = getBean("ocflIndexImpl", FedoraToOcflObjectIndex.class);
105        readOnlyTx = ReadOnlyTransaction.INSTANCE;
106        txManager = getBean(TransactionManagerImpl.class);
107    }
108
109    @Before
110    public void setUp() {
111        setBeans();
112    }
113
114    /**
115     * This test rebuilds from a known set of OCFL content.
116     * The OCFL storage root contains the following four resources:
117     * - root
118     * - /binary
119     * - /test
120     * - /test/child
121     * and a deleted object
122     * - /test/deleted-child
123     *
124     * The test verifies that these objects exist in the rebuilt repository.
125     */
126    @Test
127    public void testRebuildOcfl() {
128        rebuild("test-rebuild-ocfl/objects");
129
130        // Optional debugging
131        if (LOGGER.isDebugEnabled()) {
132            ocflRepository.listObjectIds().forEach(id -> LOGGER.debug("Object id: {}", id));
133        }
134
135        assertEquals(8, ocflRepository.listObjectIds().count());
136        assertTrue("Should contain object with id: " + FedoraTypes.FEDORA_ID_PREFIX,
137                ocflRepository.containsObject(FedoraTypes.FEDORA_ID_PREFIX));
138        assertContains("binary");
139        assertContains("test");
140        assertContains("test/child");
141        assertContains("test/deleted-child");
142        assertContains("archival-group");
143        assertContains("test/nested-archival-group");
144        assertContains("test/nested-binary");
145
146        assertNotContains("archival-group_binary");
147        assertNotContains("archival-group_container");
148        assertNotContains("junk");
149    }
150
151    @Test
152    public void testRebuildOnStart() throws Exception {
153        assertFalse("rebuild on start is disabled", fedoraPropsConfig.isRebuildOnStart());
154        rebuild("test-rebuild-ocfl/objects");
155
156        // Optional debugging
157        if (LOGGER.isDebugEnabled()) {
158            ocflRepository.listObjectIds().forEach(id -> LOGGER.debug("Object id: {}", id));
159        }
160
161        assertTrue("Should contain object with id: " + FedoraTypes.FEDORA_ID_PREFIX,
162                ocflRepository.containsObject(FedoraTypes.FEDORA_ID_PREFIX));
163        assertContains("binary");
164        assertContains("test");
165
166        final var binaryId = FedoraId.create(FedoraTypes.FEDORA_ID_PREFIX + "/binary");
167        this.ocflRepository.purgeObject(binaryId.getFullId());
168
169        //verify that the index st
170        this.index.getMapping(readOnlyTx, binaryId);
171
172        //set rebuild on start flag before initializing
173        restartContainer();
174        setBeans();
175        initializer.initialize();
176
177        //ocfl knows it is now gone
178        assertNotContains("binary");
179
180        //but the index does not know is it gone because no rebuild occurred.
181        this.index.getMapping(readOnlyTx, binaryId);
182
183        //restart the container again, but initialize after setting the rebuild on start
184        restartContainer();
185        setBeans();
186        fedoraPropsConfig.setRebuildOnStart(true);
187        initializer.initialize();
188
189        try {
190            this.index.getMapping(readOnlyTx, binaryId);
191            fail("Expected failure to retrieve mapping");
192        } catch (final FedoraOcflMappingNotFoundException ex) {
193            //intentionally left blank
194        }
195
196    }
197
198    @Test
199    public void testRebuildWebapp() throws Exception {
200        // Set how long tx will live for so that any txs created by the rebuild expire before we attempt to
201        // get the resources they created.
202        propsConfig.setSessionTimeout(Duration.ofSeconds(5));
203
204        rebuild("test-rebuild-ocfl/objects");
205
206        // Wait for txs to expire and ensure they're cleaned up
207        TimeUnit.SECONDS.sleep(5);
208        txManager.cleanupClosedTransactions();
209
210        // Test against the Fedora API
211        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("")));
212        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("test")));
213        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("binary")));
214        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("binary/" + FCR_METADATA)));
215        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("archival-group")));
216        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("archival-group/binary")));
217        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("archival-group/container")));
218        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("test/child")));
219        assertEquals(GONE.getStatusCode(), getStatus(getObjMethod("test/deleted-child")));
220        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("test/nested-archival-group")));
221        assertEquals(OK.getStatusCode(), getStatus(getObjMethod("test/nested-binary")));
222
223        final String testUri = serverAddress + "test";
224        //verify containment relationships
225        verifyContainment(serverAddress, asList("binary", "archival-group", "test"));
226        verifyContainment(serverAddress + "archival-group", asList("binary", "container"));
227        verifyContainment(testUri, asList("child", "nested-archival-group", "nested-binary"),
228                Collections.singletonList("deleted-child"));
229
230        // Get last version of test to see when deleted-child was not deleted
231        final String mementoUri;
232        try (final CloseableDataset dataset = getDataset(getObjMethod("test/" + FCR_VERSIONS))) {
233            final var graph = dataset.asDatasetGraph();
234            final var iter = graph.find(ANY, createURI(serverAddress + "test/" + FCR_VERSIONS), CONTAINS.asNode(), ANY);
235            final Iterable<Quad> iterable = () -> iter;
236            final List<String> versionList = StreamSupport.stream(iterable.spliterator(), false)
237                    .map(Quad::getObject).map(Node::getURI).sorted().collect(Collectors.toList());
238            mementoUri = versionList.get(versionList.size() - 1);
239        }
240
241        verifyContainment(mementoUri, testUri, asList("child", "nested-archival-group",
242                "nested-binary", "deleted-child"));
243    }
244
245    @Test
246    public void rebuildFailsWhenObjectFailsValidation() {
247        rebuild("test-rebuild-invalid");
248
249        assertEquals(HttpStatus.SC_NOT_FOUND, getStatus(getObjMethod("test")));
250        assertEquals(HttpStatus.SC_NOT_FOUND, getStatus(getObjMethod("binary")));
251    }
252
253    private void verifyContainment(final String subjectUri, final List<String> children) throws Exception {
254        verifyContainment(subjectUri, subjectUri, children);
255    }
256
257    private void verifyContainment(final String requestUri, final String subjectUri,
258                                   final List<String> includeChildren) throws Exception {
259        verifyContainment(requestUri, subjectUri, includeChildren, Collections.emptyList());
260    }
261
262    private void  verifyContainment(final String subjectUri, final List<String> includeChildren,
263                                    final List<String> excludeChildren) throws Exception {
264        verifyContainment(subjectUri, subjectUri, includeChildren, excludeChildren);
265    }
266
267    /**
268     * Utility to verify containment triples.
269     *
270     * @param requestUri the URI to request the graph from (could be different as in mementos or binary descriptions)
271     * @param subjectUri the URI to be the subject of the triples.
272     * @param includeChildren list of children expected, only the final path part which is appended to subjectURI.
273     * @param excludeChildren list of children not expected, only the final path part which is appended to subjectURI
274     * @throws Exception
275     */
276    private void verifyContainment(final String requestUri, final String subjectUri, final List<String> includeChildren,
277                                   final List<String> excludeChildren) throws Exception {
278        final var subjectNode = createURI(subjectUri);
279        try (final CloseableDataset dataset = getDataset(new HttpGet(requestUri))) {
280            final var graph = dataset.asDatasetGraph();
281            if (LOGGER.isDebugEnabled()) {
282                graph.listGraphNodes().forEachRemaining(gn -> {
283                    LOGGER.debug("Node = " + gn.toString());
284                });
285            }
286
287            for (final String child : includeChildren) {
288                final var childNode = createURI(subjectUri + (subjectUri.endsWith("/") ? "" : "/") + child);
289                Assert.assertTrue(format("Triple not found: {0}, {1}, {2}", subjectUri,
290                 CONTAINS, childNode),
291                        graph.contains(ANY,
292                        subjectNode,
293                        CONTAINS.asNode(),
294                        childNode));
295            }
296            for (final String child : excludeChildren) {
297                final var childNode = createURI(subjectUri + (subjectUri.endsWith("/") ? "" : "/") + child);
298                Assert.assertFalse(format("Triple found: {0}, {1}, {2}", subjectUri,
299                        CONTAINS, childNode),
300                        graph.contains(ANY,
301                                subjectNode,
302                                CONTAINS.asNode(),
303                                childNode));
304            }
305        }
306    }
307
308    private void assertContains(final String id) {
309        final var fedoraId = FedoraTypes.FEDORA_ID_PREFIX + "/" + id;
310        assertTrue("Should contain object with id: " + fedoraId,
311                ocflRepository.containsObject(fedoraId));
312    }
313
314    private void assertNotContains(final String id) {
315        final var fedoraId = FedoraTypes.FEDORA_ID_PREFIX + "/" + id;
316        assertFalse("Should NOT contain object with id: " + fedoraId,
317                ocflRepository.containsObject(fedoraId));
318    }
319
320    private void rebuild(final String name) {
321        copyToOcfl(name);
322        reindexService.reset();
323        initializer.initialize();
324    }
325
326    private void copyToOcfl(final String name) {
327        try {
328            // this is necessary so that the cache is cleared
329            ocflRepository.listObjectIds().forEach(ocflRepository::purgeObject);
330
331            try (final var list = Files.list(Paths.get("src/test/resources", name))) {
332                list.filter(Files::isDirectory).forEach(dir -> {
333                    final var objectId = URLDecoder.decode(dir.getFileName().toString(), StandardCharsets.UTF_8);
334                    var currentVersion = VersionNum.fromInt(1);
335                    var currentDir = dir.resolve(currentVersion.toString());
336
337                    while (Files.exists(currentDir)) {
338                        try {
339                            final var headers = objectMapper.readValue(
340                                    currentDir.resolve(".fcrepo/fcr-root.json").toFile(), ResourceHeaders.class);
341                            ocflRepository.putObject(ObjectVersionId.head(objectId), currentDir,
342                                    new VersionInfo().setCreated(
343                                            headers.getLastModifiedDate().atOffset(ZoneOffset.UTC)),
344                                    OcflOption.OVERWRITE);
345                            currentVersion = currentVersion.nextVersionNum();
346                            currentDir = dir.resolve(currentVersion.toString());
347                        } catch (IOException e) {
348                            throw new UncheckedIOException(e);
349                        }
350                    }
351                });
352            }
353        } catch (IOException e) {
354            throw new UncheckedIOException(e);
355        }
356    }
357
358}