Commit 7b8b445b authored by David Ronnenberg's avatar David Ronnenberg Committed by Martin Grzenia
Browse files

Add support for global views to CouchDB



This adds support for generating global design documents in
partitioned repositories. Using these design documents global queries
can be performed. In order to generate global views a new annotation
@Global is added. This annotation must be used in conjunction with
@View.

Co-authored-by: Martin Grzenia's avatarMartin Grzenia <martin.grzenia@iml.fraunhofer.de>
parent 0f9f4a25
......@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [5.3.0] - 2022-02-15
### Added
- Add support for global queries in a partitioned CouchDB repository.
For this purpose a new annotation `@Global` is added which can be used in conjunction with the `@View` annotation to mark a view as a global view.
## [5.2.1] - 2021-11-17
### Changed
- Adjust the style of license headers.
......
......@@ -11,7 +11,7 @@
<groupId>org.siliconeconomy.iotbroker</groupId>
<artifactId>sdk</artifactId>
<version>5.2.1</version>
<version>5.3.0</version>
<licenses>
<license>
......
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.View;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Marker annotation to mark a {@link View} in a {@link PartitionedRepositorySupport} as a global
* view.
* <p>
* In CouchDB, design documents in a partitioned database default to being partitioned and views
* added to partitioned design documents can only be used for queries on one partition at a time.
* For queries across multiple partitions a view in a <em>global</em> design document is required.
* {@link PartitionedRepositorySupport} creates both a partitioned design document <em>and</em> a
* global design document in a CouchDB database. In order for views to be added to the global
* design document, this annotation is to be used in conjunction with {@link View}:
* <pre>
* {@literal @Global}
* {@literal @View}(name = "myView", map = "...")
* {@literal List<TestEntity>} myQuery(...) {
* var query = createQuery("myView");
*
* // Further configuration of the query omitted...
*
* return db.queryView(query, MyEntity.class);
* }
* </pre>
* For methods that feature both annotations, the {@link PartitionedRepositorySupport} will not only
* create a corresponding view in the (default) partitioned design document but also an identical
* view in the global design document. The views in the global design document can then be used for
* global queries across multiple partitions.
*
* @author D. Ronnenberg
*/
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface Global {
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.ektorp.ComplexKey;
import org.ektorp.CouchDbConnector;
import org.ektorp.ViewQuery;
import org.ektorp.impl.StdObjectMapperFactory;
import org.ektorp.support.CouchDbRepositorySupport;
import org.ektorp.support.DesignDocument;
import java.util.HashMap;
import java.util.List;
import static java.util.Objects.requireNonNull;
/**
* Provides support for queries on global views for {@link PartitionedRepositorySupport}.
* <p>
* In order to perform global queries on partitioned databases we need this support class to
* handle the generation of a global design document. All logic concerning global queries is
* encapsulated here.
*
* @author D. Ronnenberg
*/
class GlobalRepositorySupport<T> extends CouchDbRepositorySupport<T> {
/**
* Suffix for the global design document ID.
*/
private static final String GLOBAL_DESIGN_DOC_SUFFIX = "_global";
/**
* The object mapper to use for view queries.
*/
private final ObjectMapper viewQueryObjectMapper;
protected GlobalRepositorySupport(Class<T> type, CouchDbConnector db,
String designDocName, boolean createIfNotExists,
ObjectMapper viewQueryObjectMapper) {
super(type, db, designDocName + GLOBAL_DESIGN_DOC_SUFFIX, createIfNotExists);
if (viewQueryObjectMapper == null) {
this.viewQueryObjectMapper = new StdObjectMapperFactory().createObjectMapper();
} else {
this.viewQueryObjectMapper = viewQueryObjectMapper;
}
}
public void initGlobalDesignDocument(Object realRepository) {
initDesignDocument(stdDesignDocumentId, getDesignDocumentFactory().generateGlobalFrom(realRepository));
}
/**
* If needed, initializes or updates the given design documents in the database.
*
* @param designDocId The ID of the design document in the database.
* @param generatedDefaultDDoc The content of the generated default design document.
*/
private void initDesignDocument(String designDocId, DesignDocument generatedDefaultDDoc) {
DesignDocument designDoc;
boolean update;
if (db.contains(designDocId)) {
designDoc = db.get(DesignDocument.class, designDocId);
boolean changed = designDoc.mergeWith(generatedDefaultDDoc);
// mergeWith does merge the anonymous part of the design doc (that's kinda hard). We only
// check here if we have options and merge this.
var merged = mergeOptions(generatedDefaultDDoc, designDoc);
update = changed || merged;
} else {
designDoc = generatedDefaultDDoc;
designDoc.setId(designDocId);
update = true;
}
// Update the design document only if is has changed (because e.g. views were added, or it
// did not previously exist) and it's not empty (in case a new one was created).
if (update && !isDesignDocEmpty(designDoc)) {
// This will throw an UpdateConflictException on error.
db.update(designDoc);
}
}
/**
* Compares and updates the "options" of two design documents. The options are saved in the
* anonymous part of the design document object.
*
* @param generated The generated default design document for this database.
* @param designDoc The current state of the design document - as is in the database.
* @return Indicates whether changes to designDoc has been made.
*/
public static boolean mergeOptions(DesignDocument generated, DesignDocument designDoc) {
var options = "options";
HashMap<String, Object> generatedOptions = (HashMap<String, Object>) generated.getAnonymous().getOrDefault(options, new HashMap<String, Object>());
HashMap<String, Object> existingOptions = (HashMap<String, Object>) designDoc.getAnonymous().getOrDefault(options, new HashMap<String, Object>());
requireNonNull(generatedOptions);
requireNonNull(existingOptions);
// Merge the two hashmaps. Always use the values of "generated" in case of conflict.
if (!existingOptions.equals(generatedOptions)) {
generatedOptions.forEach((key, value) -> existingOptions.merge(key, value, (v1, v2) -> v2));
designDoc.setAnonymous(options, existingOptions);
return true;
}
return false;
}
@Override
protected PartitionedDesignDocumentFactory getDesignDocumentFactory() {
return new StdGlobalDesignDocumentFactory();
}
// Global query functions. Need to be public since the shims in PartitionedRepositorySupport
// call them. That's why we need to override them here.
@Override
public ViewQuery createQuery(String viewName) {
return new ViewQuery(viewQueryObjectMapper)
.dbPath(db.path())
.designDocId(stdDesignDocumentId)
.viewName(viewName);
}
@Override
public List<T> queryView(String viewName, String key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName, int key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName, ComplexKey key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName) {
return super.queryView(viewName);
}
private boolean isDesignDocEmpty(DesignDocument designDoc) {
return designDoc.getLists().isEmpty()
&& designDoc.getViews().isEmpty()
&& designDoc.getShows().isEmpty()
&& designDoc.getUpdates().isEmpty()
&& designDoc.getFilters().isEmpty();
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.SimpleViewGenerator;
import org.ektorp.support.View;
import org.ektorp.util.ReflectionUtils;
import java.util.HashMap;
import java.util.Map;
/**
* Generate views from conjunction of {@link Global} and {@link View} annotations.
* <p>
* This class will create the needed {@link DesignDocument.View} elements from methods with
* both a {@link Global} and a {@link View} annotation. The aim is to reuse as much code as
* possible/integrate as nicely as possible with Ektorp.
*
* @author D. Ronnenberg
*/
class GlobalViewGenerator extends SimpleViewGenerator {
public Map<String, DesignDocument.View> generateGlobalViews(final Object repository) {
final Map<String, DesignDocument.View> views = new HashMap<>();
final Class<?> repositoryClass = repository.getClass();
createDeclaredGlobalViews(views, repositoryClass);
return views;
}
/**
* Create global views.
* <p>
* Main function of this class. It inspects each method of a given class looking for a
* combination of {@link View} and {@link Global} annotations. For these methods it will create
* {@link DesignDocument.View} objects.
*
* @param views the container to store the generated views in.
* @param klass the klaas that is being inspected/the views are being generated for.
*/
private void createDeclaredGlobalViews(
final Map<String, DesignDocument.View> views, final Class<?> klass) {
// This is where the magic happens.
ReflectionUtils.eachMethod(
klass,
method -> method.isAnnotationPresent(Global.class) && method.isAnnotationPresent(View.class)
).forEach(method -> addGlobalView(views, method.getAnnotation(View.class), klass));
}
// We need to provide this since SimpleViewGenerator::addView is private, thus we can't call it.
private void addGlobalView(Map<String, DesignDocument.View> views, View input,
Class<?> repositoryClass) {
// Load view from file.
if (!input.file().isEmpty()) {
views.put(input.name(), loadViewFromFile(views, input, repositoryClass));
return;
}
// Load view from class path.
if (shouldLoadFunctionFromClassPath(input.map())
|| shouldLoadFunctionFromClassPath(input.reduce())) {
views.put(input.name(), loadViewFromFile(input, repositoryClass));
return;
}
// Load view directly from the annotation - the code of the view is stored inside the
// annotation as a string.
views.put(input.name(), DesignDocument.View.of(input));
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.DesignDocumentFactory;
/**
* Marker interface for a {@link DesignDocumentFactory} with support for partitioned databases.
*
* @author D. Ronnenberg
*/
interface PartitionedDesignDocumentFactory extends DesignDocumentFactory {
/**
* Generates a global design document with views generated and loaded according to the
* annotations found in the {@code metaDataSource} object.
*
* @param metaDataSource the repository the design document is created for.
* @return the created {@link DesignDocument}.
*/
DesignDocument generateGlobalFrom(Object metaDataSource);
}
......@@ -33,16 +33,24 @@
/**
* The URI schema for partitions in partitioned databases.
*/
private static final String PARTITIONED_DESIGN_DOC_FORMAT = "_partition/%s/_design/%s";
private static final String PARTITIONED_DESIGN_DOC_QUERY_FORMAT = "_partition/%s/%s";
/**
* The name for the design document in the database. We need to save it in case the user
* provided a custom name.
* The format of the IDs for design documents in the database.
*/
private final String designDocName;
private static final String DESIGN_DOC_ID_FORMAT = "_design/%s";
/**
* The name for the partitioned design document in the database.
*/
private final String designDocId;
/**
* The object mapper to use for view queries.
*/
private final ObjectMapper viewQueryObjectMapper;
/**
* Delegate to perform global queries.
*/
private final GlobalRepositorySupport<T> globalRepositoryDelegate;
/**
* Main constructor. The interface has been slightly adjusted to accept an
......@@ -56,13 +64,20 @@
* @param viewQueryObjectMapper {@link ObjectMapper} to convert parameters of the
* {@link ViewQuery} to JSON.
*/
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
String designDocName,
boolean createIfNotExists,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName, boolean createIfNotExists,
ObjectMapper viewQueryObjectMapper) {
super(type, db, designDocName, createIfNotExists);
this.designDocName = requireNonNull(designDocName, "designDocName");
requireNonNull(designDocName, "designDocName");
// The handling (queries, ect.) of the global design doc is handled by this object.
this.globalRepositoryDelegate = new GlobalRepositorySupport<>(
type,
db,
designDocName,
false,
viewQueryObjectMapper);
this.designDocId = String.format(DESIGN_DOC_ID_FORMAT, designDocName);
if (viewQueryObjectMapper == null) {
this.viewQueryObjectMapper = new StdObjectMapperFactory().createObjectMapper();
......@@ -71,19 +86,30 @@ protected PartitionedRepositorySupport(Class<T> type,
}
}
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName,
ObjectMapper viewQueryObjectMapper) {
this(type, db, designDocName, true, viewQueryObjectMapper);
}
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName) {
this(type, db, designDocName, true, null);
}
/**
* If needed, initializes two design documents in the database (global and partitioned) or
* updates each individually.
*/
@Override
public void initStandardDesignDocument() {
// Create/update design doc with partitioned functions.
super.initStandardDesignDocument();
// Create design doc with global functions.
globalRepositoryDelegate.initGlobalDesignDocument(this);
}
/**
* Creates a {@link ViewQuery} to perform queries on partitioned databases.
*
......@@ -94,7 +120,7 @@ protected PartitionedRepositorySupport(Class<T> type,
protected ViewQuery createQuery(String viewName, String partition) {
return new ViewQuery(viewQueryObjectMapper)
.dbPath(db.path())
.designDocId(String.format(PARTITIONED_DESIGN_DOC_FORMAT, partition, designDocName))
.designDocId(String.format(PARTITIONED_DESIGN_DOC_QUERY_FORMAT, partition, designDocId))
.viewName(viewName);
}
......@@ -163,4 +189,31 @@ protected List<T> queryPartitionedView(String viewName, String partition) {
type
);
}
// Start of functions for performing global queries. These are just shims and delegate the
// work to GlobalRepositorySupport.
@Override
protected ViewQuery createQuery(String viewName) {
return globalRepositoryDelegate.createQuery(viewName);
}
@Override
protected List<T> queryView(String viewName, String key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName, int key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName, ComplexKey key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName) {
return globalRepositoryDelegate.queryView(viewName);
}
}
/*
* Copyright 2021 Open Logistics Foundation
* <p>
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.*;
import java.util.HashMap;
import java.util.Map;
/**
* Factory to create global design documents for {@link GlobalRepositorySupport}.
* <p>
* This factory will create global design documents.
*
* @author D. Ronnenberg
*/
class StdGlobalDesignDocumentFactory
extends StdDesignDocumentFactory
implements PartitionedDesignDocumentFactory {
public final GlobalViewGenerator globalViewGenerator;
public StdGlobalDesignDocumentFactory() {
this.globalViewGenerator = new GlobalViewGenerator();
}
public DesignDocument generateGlobalFrom(Object metaDataSource) {
var ddoc = newDesignDocumentInstance();
// Create views.
Map<String, DesignDocument.View> views = globalViewGenerator.generateGlobalViews(metaDataSource);
views.forEach(ddoc::addView);
// we set "partitioned" in options for this ddoc to false
var options = new HashMap<String, Object>();
options.put("partitioned", false);
ddoc.setAnonymous("options", options);
return ddoc;
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.ektorp.ViewQuery;
import org.ektorp.http.HttpClient;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.View;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.tuple;
import static org.mockito.Mockito.*;
/**
* Unit test for {@link GlobalRepositorySupport}
*
* @author D. Ronnenberg
* @author M. Grzenia
*/
class GlobalRepositorySupportTest {
/**
* Test repository using {@link TestEntity}.
*/
private static class EmptyTestRepository extends GlobalRepositorySupport<TestEntity> {
protected EmptyTestRepository(PartitionedCouchDbConnector db, String designDocName,
boolean createIfNotExists, ObjectMapper viewQueryObjectMapper) {
super(TestEntity.class, db, designDocName, createIfNotExists, viewQueryObjectMapper);
}
}
private static class TestRepositoryWithView extends GlobalRepositorySupport<TestEntity> {
protected TestRepositoryWithView(PartitionedCouchDbConnector db, String designDocName,
boolean createIfNotExists, ObjectMapper viewQueryObjectMapper) {
super(TestEntity.class, db, designDocName, createIfNotExists, viewQueryObjectMapper);
}
@Global
@View(name = "testView1", map = "mapFunction1")
public List<TestEntity> getTestView1() {
return List.of(new TestEntity());
}
}
private final static String TEST_DB_NAME = "testdb";
private final static String TEST_DB_PATH = "/" + TEST_DB_NAME + "/";
private PartitionedCouchDbConnector db;
@BeforeEach
void setUp() {
HttpClient client = mock(HttpClient.class);
db = mock(PartitionedCouchDbConnector.class);
// set up base mocks
when(db.getConnection()).thenReturn(client);
when(db.path()).thenReturn(TEST_DB_PATH);