Commit 05679155 authored by Martin Grzenia's avatar Martin Grzenia
Browse files

Merge branch 'feature/extend-couchdb-support' into 'main'

Add support for global views to CouchDB

Closes SE-5235

See merge request silicon-economy/base/iotbroker/sdk!11
parents 0f9f4a25 7b8b445b
......@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [5.3.0] - 2022-02-15
### Added
- Add support for global queries in a partitioned CouchDB repository.
For this purpose a new annotation `@Global` is added which can be used in conjunction with the `@View` annotation to mark a view as a global view.
## [5.2.1] - 2021-11-17
### Changed
- Adjust the style of license headers.
......
......@@ -11,7 +11,7 @@
<groupId>org.siliconeconomy.iotbroker</groupId>
<artifactId>sdk</artifactId>
<version>5.2.1</version>
<version>5.3.0</version>
<licenses>
<license>
......
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.View;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Marker annotation to mark a {@link View} in a {@link PartitionedRepositorySupport} as a global
* view.
* <p>
* In CouchDB, design documents in a partitioned database default to being partitioned and views
* added to partitioned design documents can only be used for queries on one partition at a time.
* For queries across multiple partitions a view in a <em>global</em> design document is required.
* {@link PartitionedRepositorySupport} creates both a partitioned design document <em>and</em> a
* global design document in a CouchDB database. In order for views to be added to the global
* design document, this annotation is to be used in conjunction with {@link View}:
* <pre>
* {@literal @Global}
* {@literal @View}(name = "myView", map = "...")
* {@literal List<TestEntity>} myQuery(...) {
* var query = createQuery("myView");
*
* // Further configuration of the query omitted...
*
* return db.queryView(query, MyEntity.class);
* }
* </pre>
* For methods that feature both annotations, the {@link PartitionedRepositorySupport} will not only
* create a corresponding view in the (default) partitioned design document but also an identical
* view in the global design document. The views in the global design document can then be used for
* global queries across multiple partitions.
*
* @author D. Ronnenberg
*/
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface Global {
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.ektorp.ComplexKey;
import org.ektorp.CouchDbConnector;
import org.ektorp.ViewQuery;
import org.ektorp.impl.StdObjectMapperFactory;
import org.ektorp.support.CouchDbRepositorySupport;
import org.ektorp.support.DesignDocument;
import java.util.HashMap;
import java.util.List;
import static java.util.Objects.requireNonNull;
/**
* Provides support for queries on global views for {@link PartitionedRepositorySupport}.
* <p>
* In order to perform global queries on partitioned databases we need this support class to
* handle the generation of a global design document. All logic concerning global queries is
* encapsulated here.
*
* @author D. Ronnenberg
*/
class GlobalRepositorySupport<T> extends CouchDbRepositorySupport<T> {
/**
* Suffix for the global design document ID.
*/
private static final String GLOBAL_DESIGN_DOC_SUFFIX = "_global";
/**
* The object mapper to use for view queries.
*/
private final ObjectMapper viewQueryObjectMapper;
protected GlobalRepositorySupport(Class<T> type, CouchDbConnector db,
String designDocName, boolean createIfNotExists,
ObjectMapper viewQueryObjectMapper) {
super(type, db, designDocName + GLOBAL_DESIGN_DOC_SUFFIX, createIfNotExists);
if (viewQueryObjectMapper == null) {
this.viewQueryObjectMapper = new StdObjectMapperFactory().createObjectMapper();
} else {
this.viewQueryObjectMapper = viewQueryObjectMapper;
}
}
public void initGlobalDesignDocument(Object realRepository) {
initDesignDocument(stdDesignDocumentId, getDesignDocumentFactory().generateGlobalFrom(realRepository));
}
/**
* If needed, initializes or updates the given design documents in the database.
*
* @param designDocId The ID of the design document in the database.
* @param generatedDefaultDDoc The content of the generated default design document.
*/
private void initDesignDocument(String designDocId, DesignDocument generatedDefaultDDoc) {
DesignDocument designDoc;
boolean update;
if (db.contains(designDocId)) {
designDoc = db.get(DesignDocument.class, designDocId);
boolean changed = designDoc.mergeWith(generatedDefaultDDoc);
// mergeWith does merge the anonymous part of the design doc (that's kinda hard). We only
// check here if we have options and merge this.
var merged = mergeOptions(generatedDefaultDDoc, designDoc);
update = changed || merged;
} else {
designDoc = generatedDefaultDDoc;
designDoc.setId(designDocId);
update = true;
}
// Update the design document only if is has changed (because e.g. views were added, or it
// did not previously exist) and it's not empty (in case a new one was created).
if (update && !isDesignDocEmpty(designDoc)) {
// This will throw an UpdateConflictException on error.
db.update(designDoc);
}
}
/**
* Compares and updates the "options" of two design documents. The options are saved in the
* anonymous part of the design document object.
*
* @param generated The generated default design document for this database.
* @param designDoc The current state of the design document - as is in the database.
* @return Indicates whether changes to designDoc has been made.
*/
public static boolean mergeOptions(DesignDocument generated, DesignDocument designDoc) {
var options = "options";
HashMap<String, Object> generatedOptions = (HashMap<String, Object>) generated.getAnonymous().getOrDefault(options, new HashMap<String, Object>());
HashMap<String, Object> existingOptions = (HashMap<String, Object>) designDoc.getAnonymous().getOrDefault(options, new HashMap<String, Object>());
requireNonNull(generatedOptions);
requireNonNull(existingOptions);
// Merge the two hashmaps. Always use the values of "generated" in case of conflict.
if (!existingOptions.equals(generatedOptions)) {
generatedOptions.forEach((key, value) -> existingOptions.merge(key, value, (v1, v2) -> v2));
designDoc.setAnonymous(options, existingOptions);
return true;
}
return false;
}
@Override
protected PartitionedDesignDocumentFactory getDesignDocumentFactory() {
return new StdGlobalDesignDocumentFactory();
}
// Global query functions. Need to be public since the shims in PartitionedRepositorySupport
// call them. That's why we need to override them here.
@Override
public ViewQuery createQuery(String viewName) {
return new ViewQuery(viewQueryObjectMapper)
.dbPath(db.path())
.designDocId(stdDesignDocumentId)
.viewName(viewName);
}
@Override
public List<T> queryView(String viewName, String key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName, int key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName, ComplexKey key) {
return super.queryView(viewName, key);
}
@Override
public List<T> queryView(String viewName) {
return super.queryView(viewName);
}
private boolean isDesignDocEmpty(DesignDocument designDoc) {
return designDoc.getLists().isEmpty()
&& designDoc.getViews().isEmpty()
&& designDoc.getShows().isEmpty()
&& designDoc.getUpdates().isEmpty()
&& designDoc.getFilters().isEmpty();
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.SimpleViewGenerator;
import org.ektorp.support.View;
import org.ektorp.util.ReflectionUtils;
import java.util.HashMap;
import java.util.Map;
/**
* Generate views from conjunction of {@link Global} and {@link View} annotations.
* <p>
* This class will create the needed {@link DesignDocument.View} elements from methods with
* both a {@link Global} and a {@link View} annotation. The aim is to reuse as much code as
* possible/integrate as nicely as possible with Ektorp.
*
* @author D. Ronnenberg
*/
class GlobalViewGenerator extends SimpleViewGenerator {
public Map<String, DesignDocument.View> generateGlobalViews(final Object repository) {
final Map<String, DesignDocument.View> views = new HashMap<>();
final Class<?> repositoryClass = repository.getClass();
createDeclaredGlobalViews(views, repositoryClass);
return views;
}
/**
* Create global views.
* <p>
* Main function of this class. It inspects each method of a given class looking for a
* combination of {@link View} and {@link Global} annotations. For these methods it will create
* {@link DesignDocument.View} objects.
*
* @param views the container to store the generated views in.
* @param klass the klaas that is being inspected/the views are being generated for.
*/
private void createDeclaredGlobalViews(
final Map<String, DesignDocument.View> views, final Class<?> klass) {
// This is where the magic happens.
ReflectionUtils.eachMethod(
klass,
method -> method.isAnnotationPresent(Global.class) && method.isAnnotationPresent(View.class)
).forEach(method -> addGlobalView(views, method.getAnnotation(View.class), klass));
}
// We need to provide this since SimpleViewGenerator::addView is private, thus we can't call it.
private void addGlobalView(Map<String, DesignDocument.View> views, View input,
Class<?> repositoryClass) {
// Load view from file.
if (!input.file().isEmpty()) {
views.put(input.name(), loadViewFromFile(views, input, repositoryClass));
return;
}
// Load view from class path.
if (shouldLoadFunctionFromClassPath(input.map())
|| shouldLoadFunctionFromClassPath(input.reduce())) {
views.put(input.name(), loadViewFromFile(input, repositoryClass));
return;
}
// Load view directly from the annotation - the code of the view is stored inside the
// annotation as a string.
views.put(input.name(), DesignDocument.View.of(input));
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.DesignDocumentFactory;
/**
* Marker interface for a {@link DesignDocumentFactory} with support for partitioned databases.
*
* @author D. Ronnenberg
*/
interface PartitionedDesignDocumentFactory extends DesignDocumentFactory {
/**
* Generates a global design document with views generated and loaded according to the
* annotations found in the {@code metaDataSource} object.
*
* @param metaDataSource the repository the design document is created for.
* @return the created {@link DesignDocument}.
*/
DesignDocument generateGlobalFrom(Object metaDataSource);
}
......@@ -33,16 +33,24 @@
/**
* The URI schema for partitions in partitioned databases.
*/
private static final String PARTITIONED_DESIGN_DOC_FORMAT = "_partition/%s/_design/%s";
private static final String PARTITIONED_DESIGN_DOC_QUERY_FORMAT = "_partition/%s/%s";
/**
* The name for the design document in the database. We need to save it in case the user
* provided a custom name.
* The format of the IDs for design documents in the database.
*/
private final String designDocName;
private static final String DESIGN_DOC_ID_FORMAT = "_design/%s";
/**
* The name for the partitioned design document in the database.
*/
private final String designDocId;
/**
* The object mapper to use for view queries.
*/
private final ObjectMapper viewQueryObjectMapper;
/**
* Delegate to perform global queries.
*/
private final GlobalRepositorySupport<T> globalRepositoryDelegate;
/**
* Main constructor. The interface has been slightly adjusted to accept an
......@@ -56,13 +64,20 @@
* @param viewQueryObjectMapper {@link ObjectMapper} to convert parameters of the
* {@link ViewQuery} to JSON.
*/
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
String designDocName,
boolean createIfNotExists,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName, boolean createIfNotExists,
ObjectMapper viewQueryObjectMapper) {
super(type, db, designDocName, createIfNotExists);
this.designDocName = requireNonNull(designDocName, "designDocName");
requireNonNull(designDocName, "designDocName");
// The handling (queries, ect.) of the global design doc is handled by this object.
this.globalRepositoryDelegate = new GlobalRepositorySupport<>(
type,
db,
designDocName,
false,
viewQueryObjectMapper);
this.designDocId = String.format(DESIGN_DOC_ID_FORMAT, designDocName);
if (viewQueryObjectMapper == null) {
this.viewQueryObjectMapper = new StdObjectMapperFactory().createObjectMapper();
......@@ -71,19 +86,30 @@ protected PartitionedRepositorySupport(Class<T> type,
}
}
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName,
ObjectMapper viewQueryObjectMapper) {
this(type, db, designDocName, true, viewQueryObjectMapper);
}
protected PartitionedRepositorySupport(Class<T> type,
PartitionedCouchDbConnector db,
protected PartitionedRepositorySupport(Class<T> type, PartitionedCouchDbConnector db,
String designDocName) {
this(type, db, designDocName, true, null);
}
/**
* If needed, initializes two design documents in the database (global and partitioned) or
* updates each individually.
*/
@Override
public void initStandardDesignDocument() {
// Create/update design doc with partitioned functions.
super.initStandardDesignDocument();
// Create design doc with global functions.
globalRepositoryDelegate.initGlobalDesignDocument(this);
}
/**
* Creates a {@link ViewQuery} to perform queries on partitioned databases.
*
......@@ -94,7 +120,7 @@ protected PartitionedRepositorySupport(Class<T> type,
protected ViewQuery createQuery(String viewName, String partition) {
return new ViewQuery(viewQueryObjectMapper)
.dbPath(db.path())
.designDocId(String.format(PARTITIONED_DESIGN_DOC_FORMAT, partition, designDocName))
.designDocId(String.format(PARTITIONED_DESIGN_DOC_QUERY_FORMAT, partition, designDocId))
.viewName(viewName);
}
......@@ -163,4 +189,31 @@ protected List<T> queryPartitionedView(String viewName, String partition) {
type
);
}
// Start of functions for performing global queries. These are just shims and delegate the
// work to GlobalRepositorySupport.
@Override
protected ViewQuery createQuery(String viewName) {
return globalRepositoryDelegate.createQuery(viewName);
}
@Override
protected List<T> queryView(String viewName, String key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName, int key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName, ComplexKey key) {
return globalRepositoryDelegate.queryView(viewName, key);
}
@Override
protected List<T> queryView(String viewName) {
return globalRepositoryDelegate.queryView(viewName);
}
}
/*
* Copyright 2021 Open Logistics Foundation
* <p>
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import org.ektorp.support.*;
import java.util.HashMap;
import java.util.Map;
/**
* Factory to create global design documents for {@link GlobalRepositorySupport}.
* <p>
* This factory will create global design documents.
*
* @author D. Ronnenberg
*/
class StdGlobalDesignDocumentFactory
extends StdDesignDocumentFactory
implements PartitionedDesignDocumentFactory {
public final GlobalViewGenerator globalViewGenerator;
public StdGlobalDesignDocumentFactory() {
this.globalViewGenerator = new GlobalViewGenerator();
}
public DesignDocument generateGlobalFrom(Object metaDataSource) {
var ddoc = newDesignDocumentInstance();
// Create views.
Map<String, DesignDocument.View> views = globalViewGenerator.generateGlobalViews(metaDataSource);
views.forEach(ddoc::addView);
// we set "partitioned" in options for this ddoc to false
var options = new HashMap<String, Object>();
options.put("partitioned", false);
ddoc.setAnonymous("options", options);
return ddoc;
}
}
/*
* Copyright 2021 Open Logistics Foundation
*
* Licensed under the Open Logistics License 1.0.
* For details on the licensing terms, see the LICENSE file.
*/
package org.siliconeconomy.iotbroker.utils.couchdb;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.ektorp.ViewQuery;
import org.ektorp.http.HttpClient;
import org.ektorp.support.DesignDocument;
import org.ektorp.support.View;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.tuple;
import static org.mockito.Mockito.*;
/**
* Unit test for {@link GlobalRepositorySupport}
*
* @author D. Ronnenberg
* @author M. Grzenia
*/
class GlobalRepositorySupportTest {
/**
* Test repository using {@link TestEntity}.
*/
private static class EmptyTestRepository extends GlobalRepositorySupport<TestEntity> {
protected EmptyTestRepository(PartitionedCouchDbConnector db, String designDocName,
boolean createIfNotExists, ObjectMapper viewQueryObjectMapper) {
super(TestEntity.class, db, designDocName, createIfNotExists, viewQueryObjectMapper);
}
}
private static class TestRepositoryWithView extends GlobalRepositorySupport<TestEntity> {
protected TestRepositoryWithView(PartitionedCouchDbConnector db, String designDocName,
boolean createIfNotExists, ObjectMapper viewQueryObjectMapper) {
super(TestEntity.class, db, designDocName, createIfNotExists, viewQueryObjectMapper);
}
@Global
@View(name = "testView1", map = "mapFunction1")
public List<TestEntity> getTestView1() {
return List.of(new TestEntity());
}
}
private final static String TEST_DB_NAME = "testdb";
private final static String TEST_DB_PATH = "/" + TEST_DB_NAME + "/";
private PartitionedCouchDbConnector db;
@BeforeEach
void setUp() {
HttpClient client = mock(HttpClient.class);
db = mock(PartitionedCouchDbConnector.class);
// set up base mocks
when(db.getConnection()).thenReturn(client);
when(db.path()).thenReturn(TEST_DB_PATH);
}