Commit 82b39507 authored by bseeger's avatar bseeger
Browse files

Merge branch 'ore' into 'master'

Add ORE extension

Similar to the existing PCDM extension, this assembles an ORE Aggregation into a single graph with a single HTTP request to the service.

See merge request !93
parents 37ed67fa bc8be765
......@@ -16,6 +16,7 @@ by making available a REST-based HTTP interface. The intention is that these ext
* [`acrepo-exts-fits`](acrepo-exts-fits): This service will return FITS information associated with a Fedora Binary, in XML format
* [`acrepo-exts-image`](acrepo-exts-image): An image manipulation service
* [`acrepo-exts-ldpath`](acrepo-exts-ldpath): This module extends `fcrepo-ldpath` to support additional Linked Data endpoints (e.g. Getty)
* [`acrepo-exts-ore`](acrepo-exts-ore): This constructs a complete ORE Aggregation graph for Fedora resources
* [`acrepo-exts-pcdm`](acrepo-exts-pcdm): This constructs a complete PCDM object graph for Fedora resources
* [`acrepo-exts-serialize-xml`](acrepo-exts-serialize-xml): This service translates Fedora RDF documents into MODS/XML or DC/XML
* [`acrepo-exts-template`](acrepo-exts-template): A module for converting Fedora resources into some other form, using a [mustache](https://mustache.github.io/) template.
......@@ -27,6 +28,7 @@ These modules provide particular services, independent of Fedora Resources.
* [`acrepo-services-inference`](acrepo-services-inference): An OSGi-based structural typing service using owl inference
* [`acrepo-services-mint`](acrepo-services-mint): This mints random (public) URIs for use with Fedora resources
* [`acrepo-services-ore`](acrepo-services-ore): This makes it easy to work with ORE aggregations
* [`acrepo-services-pcdm`](acrepo-services-pcdm): This makes it easy to work with PCDM objects
Connectors
......@@ -79,12 +81,14 @@ command from its shell:
feature:install acrepo-exts-fits
feature:install acrepo-exts-image
feature:install acrepo-exts-ldpath
feature:install acrepo-exts-ore
feature:install acrepo-exts-pcdm
feature:install acrepo-exts-serialize-xml
feature:install acrepo-exts-template
feature:install acrepo-services-inference
feature:install acrepo-services-mint
feature:install acrepo-services-ore
feature:install acrepo-services-pcdm
More information
......
Repository ORE object extension
================================
This extension operates on ORE Aggregations, building an RDF graph of
the complete object (following `ore:aggregates` links). The complete
graph is returned in the requested serialization, using an `Accept` header.
For example:
curl http://localhost:9108/ore?context=http://localhost:8080/fcrepo/rest/a/b/c
Building
--------
To build this project use
gradle install
Deploying in OSGi
-----------------
This projects can be deployed in an OSGi container. For example using
[Apache Karaf](http://karaf.apache.org) version 4.x or better, you can run the following
command from its shell:
feature:repo-add mvn:edu.amherst.acdc/acrepo-karaf/LATEST/xml/features
feature:install acrepo-exts-ore
feature:install acrepo-services-ore
Configuration
-------------
The application can be configured by creating the following configuration
file `$KARAF_HOME/etc/edu.amherst.acdc.exts.ore.cfg`. The following values
are available for configuration:
The base url of the fedora repository
fcrepo.baseUrl=localhost:8080/fcrepo/rest
The port on which the service is made availalbe
rest.port=9108
The hostname for the service
rest.host=localhost
The REST prefix
rest.prefix=/ore
By editing this file, any currently running routes will be immediately redeployed
with the new values.
For more help see the [Apache Camel](http://camel.apache.org) documentation
apply plugin: 'osgi'
description = 'ORE Object builder'
dependencies {
compile group: 'org.apache.camel', name: 'camel-core', version: camelVersion
compile group: 'org.apache.camel', name: 'camel-blueprint', version: camelVersion
compile group: 'org.apache.camel', name: 'camel-jetty9', version: camelVersion
compile(group: 'org.fcrepo.camel', name: 'fcrepo-camel', version: fcrepoCamelVersion) {
exclude(module: 'slf4j-log4j12')
}
compile group: 'org.apache.jena', name: 'jena-osgi', version: jenaVersion
compile project(':acrepo-services-ore')
testCompile group: 'junit', name: 'junit', version: junitVersion
testCompile group: 'org.apache.camel', name: 'camel-test-blueprint', version: camelVersion
testCompile group: 'ch.qos.logback', name: 'logback-classic', version: logbackVersion
}
jar {
manifest {
description project.description
docURL project.docURL
vendor project.vendor
license project.license
instruction 'Import-Package', "org.apache.camel,org.fcrepo.camel,edu.amherst.acdc.services.ore,${defaultOsgiImports}"
instruction 'Export-Package', "edu.amherst.acdc.exts.ore;version=${projectOsgiVersion}"
}
}
artifacts {
archives (file('build/cfg/main/edu.amherst.acdc.exts.ore.cfg')) {
classifier 'configuration'
type 'cfg'
}
}
# HTTP Port
rest.host=localhost
rest.port=9108
rest.prefix=/ore
# Concurrency level (this MUST be > 1)
ore.concurrency=10
# Repository Base URL (it MUST start with http:// or https://)
fcrepo.baseUrl=http://localhost:8080/fcrepo/rest
fcrepo.authUsername=
fcrepo.authPassword=
# API-X configuration
extension.load=true
extension.load.uri=http://apix/services//apix:load
extension.load.maximumRedeliveries=60
/*
* Copyright 2016 Amherst College
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.amherst.acdc.exts.ore;
import static edu.amherst.acdc.exts.ore.OreHeaders.ORE_MODEL;
import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel;
import org.apache.camel.Exchange;
import org.apache.camel.processor.aggregate.AggregationStrategy;
import org.apache.jena.rdf.model.Model;
/**
* Aggregate the CamelOreModel header values across exchanges
*
* @author acoburn
* @since 6/29/16
*/
class ModelAggregator implements AggregationStrategy {
@Override
public Exchange aggregate(final Exchange a, final Exchange b) {
if (a == null) {
return b;
}
final Model modelA = a.getIn().getHeader(ORE_MODEL, Model.class);
final Model modelB = b.getIn().getHeader(ORE_MODEL, Model.class);
if (modelA == null && modelB == null) {
a.getIn().setHeader(ORE_MODEL, createDefaultModel());
} else if (modelA == null) {
a.getIn().setHeader(ORE_MODEL, modelB);
} else if (modelB != null) {
modelA.add(modelB);
a.getIn().setHeader(ORE_MODEL, modelA);
}
return a;
}
}
/*
* Copyright 2016 Amherst College
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.amherst.acdc.exts.ore;
/**
* Some header field definitions
*
* @author acoburn
*/
final class OreHeaders {
public final static String ORE_ACCEPT = "CamelOreAccept";
public final static String ORE_MODEL = "CamelOreModel";
public final static String ORE_SUBJECT = "CamelOreSubject";
private OreHeaders() {
// prevent instantiation
}
}
/*
* Copyright 2016 Amherst College
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.amherst.acdc.exts.ore;
import static edu.amherst.acdc.exts.ore.OreHeaders.ORE_ACCEPT;
import static edu.amherst.acdc.exts.ore.OreHeaders.ORE_SUBJECT;
import static java.util.Optional.ofNullable;
import static org.apache.camel.Exchange.CONTENT_TYPE;
import static org.apache.camel.Exchange.HTTP_METHOD;
import static org.apache.camel.Exchange.HTTP_RESPONSE_CODE;
import static org.fcrepo.camel.FcrepoHeaders.FCREPO_BASE_URL;
import static org.fcrepo.camel.FcrepoHeaders.FCREPO_URI;
import java.util.Optional;
import org.apache.camel.builder.RouteBuilder;
import org.apache.jena.riot.Lang;
import org.apache.jena.riot.RDFLanguages;
/**
* A content router for handling ORE extension requests
*
* @author Aaron Coburn
*/
public class OreRouter extends RouteBuilder {
private final static String DEFAULT_CONTENT_TYPE = "text/turtle";
private final static String HTTP_QUERY_CONTEXT = "context";
/**
* Configure the message route workflow.
*/
public void configure() throws Exception {
from("jetty:http://{{rest.host}}:{{rest.port}}{{rest.prefix}}?" +
"sendServerVersion=false&httpMethodRestrict=GET,OPTIONS")
.routeId("OreRouter")
.removeHeader("User-Agent")
.process(e -> e.getIn().setHeader(FCREPO_URI,
e.getIn().getHeader(HTTP_QUERY_CONTEXT,
e.getIn().getHeader("Apix-Ldp-Resource"))))
.setHeader(FCREPO_BASE_URL).simple("{{fcrepo.baseUrl}}")
.choice()
.when(header(HTTP_METHOD).isEqualTo("OPTIONS"))
.setHeader(CONTENT_TYPE).constant("text/turtle")
.setHeader("Allow").constant("GET,OPTIONS")
.to("language:simple:resource:classpath:options.ttl")
.when(header(HTTP_QUERY_CONTEXT).startsWith(header(FCREPO_BASE_URL)))
.to("direct:get");
from("direct:get").routeId("OreGet")
.setHeader(ORE_ACCEPT, header("Accept"))
.log("Building ORE Object ${body}")
.setBody().header(FCREPO_URI)
.to("seda:recurse")
.removeHeader("breadcrumbId")
.process(exchange -> {
final String contentType = exchange.getIn().getHeader(ORE_ACCEPT, String.class);
final Optional<String> rdfLang = ofNullable(contentType).map(RDFLanguages::contentTypeToLang)
.map(Lang::getName);
exchange.getIn().setHeader(CONTENT_TYPE, rdfLang.isPresent() ? contentType : DEFAULT_CONTENT_TYPE);
})
.to("direct:serialize");
from("seda:recurse?concurrentConsumers={{ore.concurrency}}").routeId("OreBuildRecursive")
.setHeader(FCREPO_URI, body())
.to("direct:getResource")
.filter(header(HTTP_RESPONSE_CODE).isEqualTo(200))
.log("Getting related resources for ${headers[CamelFcrepoUri]}")
.to("direct:parse")
.setHeader(ORE_SUBJECT).header(FCREPO_URI)
.to("direct:members")
.split(body(), new ModelAggregator())
.to("seda:recurse");
from("direct:getResource").routeId("OreResource")
.removeHeader("breadcrumbId")
.removeHeader("Accept")
.to("fcrepo:{{fcrepo.baseUrl}}?throwExceptionOnFailure=false");
}
}
<?xml version="1.0" encoding="UTF-8"?>
<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:cm="http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.1.0"
xsi:schemaLocation="
http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd
http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.1.0 http://aries.apache.org/schemas/blueprint-cm/blueprint-cm-1.1.0.xsd
http://camel.apache.org/schema/blueprint http://camel.apache.org/schema/blueprint/camel-blueprint.xsd">
<!-- OSGI blueprint property placeholder -->
<cm:property-placeholder persistent-id="edu.amherst.acdc.exts.ore" update-strategy="reload">
<cm:default-properties>
<cm:property name="rest.port" value="9108"/>
<cm:property name="rest.prefix" value="/ore"/>
<cm:property name="rest.host" value="localhost"/>
<cm:property name="ore.concurrency" value="10"/>
<cm:property name="fcrepo.baseUrl" value="http://localhost:8080/fcrepo/rest"/>
<cm:property name="fcrepo.authUsername" value=""/>
<cm:property name="fcrepo.authPassword" value=""/>
<cm:property name="extension.load" value="true" />
<cm:property name="extension.load.uri" value="http://apix/services//apix:load" />
<cm:property name="extension.load.maximumRedeliveries" value="60" />
</cm:default-properties>
</cm:property-placeholder>
<reference id="oreService" interface="edu.amherst.acdc.services.ore.OreService" filter="(osgi.jndi.service.name=acrepo/Ore)" />
<bean id="fcrepo" class="org.fcrepo.camel.FcrepoComponent">
<property name="authUsername" value="${fcrepo.authUsername}"/>
<property name="authPassword" value="${fcrepo.authPassword}"/>
<property name="baseUrl" value="${fcrepo.baseUrl}"/>
</bean>
<camelContext id="AcrepoExtOre" xmlns="http://camel.apache.org/schema/blueprint">
<package>edu.amherst.acdc.exts.ore</package>
<route id="OreParser">
<from uri="direct:parse"/>
<setHeader headerName="CamelOreModel">
<method ref="oreService" method="parseInto(${header[CamelOreModel]}, ${body}, ${header[Content-Type]})"/>
</setHeader>
</route>
<route id="OreMembers">
<from uri="direct:members"/>
<setBody>
<method ref="oreService" method="aggregates(${header[CamelOreModel]}, ${header[CamelOreSubject]})"/>
</setBody>
</route>
<route id="OreSerialize">
<from uri="direct:serialize"/>
<setBody>
<method ref="oreService" method="serialize(${header[CamelOreModel]}, ${header[CamelOreAccept]})"/>
</setBody>
</route>
<!-- Self-register the loader service as an extension -->
<route id="load-extension">
<from uri="timer:register?repeatCount=1" />
<onException>
<exception>java.lang.Exception</exception>
<redeliveryPolicy maximumRedeliveries="{{extension.load.maximumRedeliveries}}"
logRetryAttempted="true" retryAttemptedLogLevel="INFO" />
</onException>
<setHeader headerName="Content-Type">
<constant>text/plain</constant>
</setHeader>
<setHeader headerName="CamelHttpMethod">
<constant>POST</constant>
</setHeader>
<setBody>
<simple>http://{{rest.host}}:{{rest.port}}{{rest.prefix}}</simple>
</setBody>
<choice>
<when>
<simple>{{extension.load}}</simple>
<to uri="jetty:{{extension.load.uri}}?okStatusCodeRange=200-399" />
</when>
</choice>
</route>
</camelContext>
</blueprint>
@prefix owl:<http://www.w3.org/2002/07/owl#> .
@prefix rdfs:<http://www.w3.org/2000/01/rdf-schema#> .
@prefix registry:<http://acdc.amherst.edu/ns/registry#> .
@prefix ldp:<http://www.w3.org/ns/ldp#> .
@prefix apix:<http://fedora.info/definitions/v4/api-extension#> .
<> a apix:Extension;
rdfs:label "ORE Object extension";
rdfs:comment "An extension that builds an entire ORE Aggregation";
apix:exposesService registry:OreService;
apix:exposesServiceAt "svc:ore";
apix:bindsTo ldp:RDFSource .
......@@ -50,9 +50,11 @@ dependencies {
testCompile project(':acrepo-connector-idiomatic-pgsql')
testCompile project(':acrepo-services-inference')
testCompile project(':acrepo-services-mint')
testCompile project(':acrepo-services-ore')
testCompile project(':acrepo-services-pcdm')
testCompile project(':acrepo-exts-fits')
testCompile project(':acrepo-exts-image')
testCompile project(':acrepo-exts-ore')
testCompile project(':acrepo-exts-pcdm')
testCompile project(':acrepo-exts-serialize-xml')
testCompile project(':acrepo-exts-template')
......@@ -137,6 +139,7 @@ test {
systemProperty 'karaf.rmiRegistry.port', randomPort()
systemProperty 'karaf.template.port', randomPort()
systemProperty 'karaf.metadata.port', randomPort()
systemProperty 'karaf.ore.port', randomPort()
systemProperty 'karaf.pcdm.port', randomPort()
systemProperty 'karaf.image.port', randomPort()
systemProperty 'karaf.idiomatic.port', randomPort()
......
/*
* Copyright 2016 Amherst College
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.amherst.acdc.itests;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.ops4j.pax.exam.CoreOptions.maven;
import static org.ops4j.pax.exam.CoreOptions.systemProperty;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder;
import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel;
import static org.slf4j.LoggerFactory.getLogger;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.Set;
import org.apache.camel.CamelContext;
import org.apache.jena.rdf.model.Model;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.ops4j.pax.exam.Configuration;
import org.ops4j.pax.exam.ConfigurationManager;
import org.ops4j.pax.exam.Option;
import org.ops4j.pax.exam.junit.PaxExam;
import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel;
import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
import org.ops4j.pax.exam.spi.reactors.PerClass;
import org.slf4j.Logger;
/**
* @author Aaron Coburn
* @since May 2, 2016
*/
@RunWith(PaxExam.class)
@ExamReactorStrategy(PerClass.class)
public class AcrepoOreIT extends AbstractOSGiIT {
private static Logger LOGGER = getLogger(AcrepoOreIT.class);
@Configuration
public Option[] config() {
final ConfigurationManager cm = new ConfigurationManager();
final String fcrepoPort = cm.getProperty("fcrepo.dynamic.test.port");
final String oreExtPort = cm.getProperty("karaf.ore.port");
final String rmiRegistryPort = cm.getProperty("karaf.rmiRegistry.port");
final String rmiServerPort = cm.getProperty("karaf.rmiServer.port");
final String fcrepoBaseUrl = "http://localhost:" + fcrepoPort + "/fcrepo/rest";
final String sshPort = cm.getProperty("karaf.ssh.port");
return new Option[] {
karafDistributionConfiguration()
.frameworkUrl(maven().groupId("org.apache.karaf").artifactId("apache-karaf")
.version(cm.getProperty("karaf.version")).type("zip"))
.unpackDirectory(new File("build", "exam"))
.useDeployFolder(false),
logLevel(LogLevel.INFO),
keepRuntimeFolder(),
configureConsole().ignoreLocalConsole(),
features(maven().groupId("org.apache.karaf.features").artifactId("standard")
.versionAsInProject().classifier("features").type("xml"), "scr"),
features(maven().groupId("org.apache.camel.karaf").artifactId("apache-camel")
.type("xml").classifier("features").versionAsInProject()),
features(maven().groupId("org.fcrepo.camel").artifactId("fcrepo-camel")
.type("xml").classifier("features").versionAsInProject()),
features(maven().groupId("edu.amherst.acdc").artifactId("acrepo-karaf")
.type("xml").classifier("features").versionAsInProject(),
"acrepo-exts-ore"),
systemProperty("karaf.ore.port").value(oreExtPort),
systemProperty("fcrepo.port").value(fcrepoPort),
editConfigurationFilePut("etc/org.apache.karaf.management.cfg", "rmiRegistryPort", rmiRegistryPort),
editConfigurationFilePut("etc/org.apache.karaf.management.cfg", "rmiServerPort", rmiServerPort),
editConfigurationFilePut("etc/org.apache.karaf.shell.cfg", "sshPort", sshPort),
editConfigurationFilePut("etc/edu.amherst.acdc.exts.ore.cfg", "fcrepo.baseUrl", fcrepoBaseUrl),
editConfigurationFilePut("etc/edu.amherst.acdc.exts.ore.cfg", "rest.port", oreExtPort),
editConfigurationFilePut("etc/edu.amherst.acdc.exts.ore.cfg", "extension.load", "false")
};
}
@Test
public void testInstallation() throws Exception {
assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-core")));
assertTrue(featuresService.isInstalled(featuresService.getFeature("fcrepo-camel")));
assertTrue(featuresService.isInstalled(featuresService.getFeature("acrepo-exts-ore")));
assertTrue(featuresService.isInstalled(featuresService.getFeature("acrepo-services-ore")));
}
@Test
public void testOptions() throws Exception {
// make sure that the camel context has started up.
final CamelContext ctx = getOsgiService(CamelContext.class, "(camel.context.name=AcrepoExtOre)",
10000);
assertNotNull(ctx);
final String baseUrl = "http://localhost:" + System.getProperty("fcrepo.port") + "/fcrepo/rest";
final String baseSvcUrl = "http://localhost:" + System.getProperty("karaf.ore.port") + "/ore";
assertTrue(options(baseSvcUrl).contains("apix:bindsTo ldp:RDFSource"));
}
@Test
public void testOreObjectTurtle() throws Exception {
// make sure that the camel context has started up.
final CamelContext ctx = getOsgiService(CamelContext.class, "(camel.context.name=AcrepoExtOre)",
10000);
assertNotNull(ctx);
final String baseUrl = "http://localhost:" + System.getProperty("fcrepo.port") + "/fcrepo/rest";
final String baseSvcUrl = "http://localhost:" + System.getProperty("karaf.ore.port") + "/ore";
final String oreObj = post(baseUrl, getClass().getResourceAsStream("/aggregation.ttl"), "text/turtle");
final String members = oreObj + "/members";
assertTrue(put(members, getClass().getResourceAsStream("/directcontainer.ttl"), "text/turtle"));
final String page1 = post(members);
final String page2 = post(members);