diff --git a/specifyweb/businessrules/uniqueness_rules.py b/specifyweb/businessrules/uniqueness_rules.py index 4479f5d846d..239118524f3 100644 --- a/specifyweb/businessrules/uniqueness_rules.py +++ b/specifyweb/businessrules/uniqueness_rules.py @@ -71,6 +71,9 @@ def check_unique(instance): 'Collectionobject': { 'catalognumber': ['collection'], }, + 'Collectionrelationship' : { + 'rightside' : ['collectionreltype'] + }, 'Collector': { 'agent': ['collectingevent'], }, diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts b/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts index 9872f16e56b..bce230c54b9 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts @@ -110,6 +110,12 @@ const globalFieldOverrides: { Attachment: { tableID: 'optional', }, + CollectionRelationship: { + collectionRelType: 'required', + }, + CollectionRelType: { + name: 'required', + }, Taxon: { parent: 'required', isAccepted: 'readOnly', diff --git a/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx b/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx index 98521427efa..4078514add8 100644 --- a/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx @@ -162,7 +162,7 @@ export function QueryComboBox({ typeof resource.getDependentResource(field.name) === 'object') ? resource .rgetPromise(field.name) - .then((resource) => + .then(async (resource) => resource === undefined || resource === null ? { label: '' as LocalizedString, diff --git a/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx b/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx index c948e0ff0ed..48d82b3568a 100644 --- a/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx @@ -18,6 +18,7 @@ import { PrintOnSave } from '../FormFields/Checkbox'; import type { ViewDescription } from '../FormParse'; import { SubViewContext } from '../Forms/SubView'; import { isTreeResource } from '../InitialContext/treeRanks'; +import { interactionTables } from '../Interactions/config'; import { Dialog } from '../Molecules/Dialog'; import { ProtectedAction, @@ -33,7 +34,6 @@ import { QueryTreeUsages } from './QueryTreeUsages'; import { ReadOnlyMode } from './ReadOnlyMode'; import { ShareRecord } from './ShareRecord'; import { SubViewMeta } from './SubViewMeta'; -import { interactionTables } from '../Interactions/config'; /** * Form preferences host context aware user preferences and other meta-actions. diff --git a/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts b/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts index 159d8c70672..e51df316e4f 100644 --- a/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts +++ b/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts @@ -57,6 +57,18 @@ export const webOnlyViews = f.store(() => 'spReports', ]) ), + CollectionRelType: autoGenerateViewDefinition( + schema.models.CollectionRelType, + 'form', + 'edit', + ['name', 'leftSideCollection', 'rightSideCollection', 'remarks'] + ), + CollectionRelationship: autoGenerateViewDefinition( + schema.models.CollectionRelationship, + 'form', + 'edit', + ['collectionRelType', 'leftSide', 'rightSide'] + ), [spAppResourceView]: autoGenerateViewDefinition( schema.models.SpAppResource, 'form', diff --git a/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx b/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx index 8c5535e95f0..ac3548fcd68 100644 --- a/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx @@ -359,7 +359,7 @@ function RecordSet({ }).then(({ totalCount }) => totalCount !== 0), }) ) - ).then((results) => { + ).then(async (results) => { const [nonDuplicates, duplicates] = split( results, ({ isDuplicate }) => isDuplicate diff --git a/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts b/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts index ce2d46d907c..00c29998abb 100644 --- a/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts +++ b/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts @@ -202,6 +202,12 @@ export const getFrontEndPickLists = f.store<{ .set('tableName', 'preptype') .set('fieldName', 'name'), }, + CollectionRelType: { + name: definePicklist('_CollectionRelType', []) + .set('type', PickListTypes.FIELDS) + .set('tableName', 'collectionreltype') + .set('fieldName', 'name'), + }, SpAppResource: { mimeType: definePicklist( '_MimeType', diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx index daf2f946dae..4a0a1c2a3f1 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx @@ -6,7 +6,8 @@ import type { RA } from '../../utils/types'; import { ensure } from '../../utils/types'; import { error } from '../Errors/assert'; import type { StatLayout } from '../Statistics/types'; -import { GenericPreferences, defineItem } from './types'; +import type { GenericPreferences } from './types'; +import { defineItem } from './types'; export const collectionPreferenceDefinitions = { statistics: { diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx index b530c261058..ef95e189ebc 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx @@ -1,11 +1,11 @@ import React from 'react'; +import { useLiveState } from '../../hooks/useLiveState'; +import type { AppResourceTab } from '../AppResources/TabDefinitions'; import { PreferencesContent } from '../Preferences'; import { BasePreferences } from '../Preferences/BasePreferences'; import { userPreferenceDefinitions } from '../Preferences/UserDefinitions'; import { userPreferences } from '../Preferences/userPreferences'; -import { AppResourceTab } from '../AppResources/TabDefinitions'; -import { useLiveState } from '../../hooks/useLiveState'; export const UserPreferencesEditor: AppResourceTab = function ({ isReadOnly, diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx index ef782a81ecc..c20f4f0a773 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx @@ -31,9 +31,9 @@ import { rawMenuItemsPromise } from '../Header/menuItemDefinitions'; import { useMenuItems, useUserTools } from '../Header/menuItemProcessing'; import { AttachmentPicker } from '../Molecules/AttachmentPicker'; import { AutoComplete } from '../Molecules/AutoComplete'; -import { userPreferences } from './userPreferences'; import { ListEdit } from '../Toolbar/QueryTablesEdit'; -import { PreferenceItem, PreferenceItemComponent } from './types'; +import type { PreferenceItem, PreferenceItemComponent } from './types'; +import { userPreferences } from './userPreferences'; export const ColorPickerPreferenceItem: PreferenceItemComponent = function ColorPickerPreferenceItem({ diff --git a/specifyweb/frontend/js_src/lib/components/Security/User.tsx b/specifyweb/frontend/js_src/lib/components/Security/User.tsx index e343515cfe6..398ffc343f9 100644 --- a/specifyweb/frontend/js_src/lib/components/Security/User.tsx +++ b/specifyweb/frontend/js_src/lib/components/Security/User.tsx @@ -468,7 +468,7 @@ function UserView({ status: Http.NO_CONTENT, }) ) - .then(({ data, status }) => + .then(async ({ data, status }) => status === Http.BAD_REQUEST ? setState({ type: 'SettingAgents', @@ -515,7 +515,7 @@ function UserView({ }) : true ) - .then((canContinue) => + .then(async (canContinue) => canContinue === true ? Promise.all([ typeof password === 'string' && password !== '' diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts index 0e7101bd1b7..7aa22c6671a 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts @@ -3,8 +3,8 @@ import { theories } from '../../../tests/utils'; import type { RA } from '../../../utils/types'; import type { AutoMapperResults } from '../autoMapper'; import { - AutoMapper as AutoMapperConstructor, type AutoMapperConstructorParameters, + AutoMapper as AutoMapperConstructor, circularTables, } from '../autoMapper'; diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index c3f252d61df..bee5dfe09e3 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, Union, Callable +from typing import Dict, Any, Optional, Tuple, Callable, Union from specifyweb.specify.datamodel import datamodel, Table, Relationship from specifyweb.specify.load_datamodel import DoesNotExistError @@ -7,11 +7,38 @@ from specifyweb.stored_queries.format import get_date_format from .uploadable import Uploadable, ScopedUploadable -from .upload_table import UploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable +from .upload_table import UploadTable, DeferredScopeUploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable from .tomany import ToManyRecord, ScopedToManyRecord from .treerecord import TreeRecord, ScopedTreeRecord from .column_options import ColumnOptions, ExtendedColumnOptions +""" There are cases in which the scoping of records should be dependent on another record/column in a WorkBench dataset. + +The DEFERRED_SCOPING dictonary defines the information needed to extract the correct scope to upload/validate a record into. + +The structure of DEFERRED_SCOPING is as following: + The keys are tuples containing the django table name and a relationship that should be scoped. + + The values are tuples containing the table name, field to filter on, and value to pull from that field to use as the collection for the + tableName.fieldName in the associated key of DEFERRED_SCOPING + + For example, consider the following the deferred scoping information: + ("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection') + + This information describes the following process to be performed: + + 'when uploading the rightside of a Collection Relationship, get the Collection Rel Type in the database from the dataset by + filtering Collection Rel Types in the database by name. Then, set the collection of the Collectionrelationship rightside equal to the Collection Rel Type's + rightSideCollection' + + See .upload_plan_schema.py for how this is used + +""" +DEFERRED_SCOPING: Dict[Tuple[str, str], Tuple[str, str, str]] = { + ("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection'), + ("Collectionrelationship", "leftside"): ('collectionreltype', 'name', 'leftsidecollection'), + } + def scoping_relationships(collection, table: Table) -> Dict[str, int]: extra_static: Dict[str, int] = {} @@ -80,11 +107,14 @@ def extend_columnoptions(colopts: ColumnOptions, collection, tablename: str, fie dateformat=get_date_format(), ) - -def apply_scoping_to_uploadtable(ut: UploadTable, collection) -> ScopedUploadTable: +def apply_scoping_to_uploadtable(ut: Union[UploadTable, DeferredScopeUploadTable], collection) -> ScopedUploadTable: table = datamodel.get_table_strict(ut.name) adjust_to_ones = to_one_adjustments(collection, table) + + if ut.overrideScope is not None and isinstance(ut.overrideScope['collection'], int): + collection = getattr(models, "Collection").objects.filter(id=ut.overrideScope['collection']).get() + return ScopedUploadTable( name=ut.name, diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 2b6381354f2..7aabb873bcb 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -140,6 +140,7 @@ def with_scoping(collection) -> ScopedUploadTable: wbcols = { 'catalognumber' : parse_column_options("BMSM No."), }, + overrideScope=None, static = {}, toMany = { 'determinations': [ @@ -160,6 +161,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial': parse_column_options('Determiner 1 Middle Initial'), 'lastname': parse_column_options('Determiner 1 Last Name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, @@ -188,6 +190,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'startdate' : parse_column_options('Start Date Collected'), 'stationfieldnumber' : parse_column_options('Station No.'), }, + overrideScope=None, static = {}, toOne = { 'locality': UploadTable( @@ -197,6 +200,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'latitude1': parse_column_options('Latitude1'), 'longitude1': parse_column_options('Longitude1'), }, + overrideScope=None, static = {'srclatlongunit': 0}, toOne = { 'geography': TreeRecord( @@ -227,6 +231,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial' : parse_column_options('Collector 1 Middle Initial'), 'lastname' : parse_column_options('Collector 1 Last Name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, @@ -246,6 +251,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial' : parse_column_options('Collector 2 Middle Initial'), 'lastname' : parse_column_options('Collector 2 Last name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, diff --git a/specifyweb/workbench/upload/tests/testdisambiguation.py b/specifyweb/workbench/upload/tests/testdisambiguation.py index fafa2ea6c72..24ff86b7556 100644 --- a/specifyweb/workbench/upload/tests/testdisambiguation.py +++ b/specifyweb/workbench/upload/tests/testdisambiguation.py @@ -31,6 +31,7 @@ def test_disambiguation(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -43,6 +44,7 @@ def test_disambiguation(self) -> None: wbcols={ 'lastname': parse_column_options('author1') }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -56,6 +58,7 @@ def test_disambiguation(self) -> None: wbcols={ 'lastname': parse_column_options('author2') }, + overrideScope=None, static={}, toOne={}, toMany={} diff --git a/specifyweb/workbench/upload/tests/testparsing.py b/specifyweb/workbench/upload/tests/testparsing.py index ccbd7fb3189..d8156817035 100644 --- a/specifyweb/workbench/upload/tests/testparsing.py +++ b/specifyweb/workbench/upload/tests/testparsing.py @@ -143,6 +143,7 @@ def test_nonreadonly_picklist(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno'), 'text1': parse_column_options('habitat')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -185,6 +186,7 @@ def test_uiformatter_match(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -210,6 +212,7 @@ def test_numeric_types(self) -> None: 'number1': parse_column_options('float'), 'totalvalue': parse_column_options('decimal') }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -231,6 +234,7 @@ def test_required_field(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno'), 'text1': parse_column_options('habitat')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -253,6 +257,7 @@ def test_readonly_picklist(self) -> None: 'title': parse_column_options('title'), 'lastname': parse_column_options('lastname'), }, + overrideScope=None, static={'agenttype': 1}, toOne={}, toMany={} @@ -358,6 +363,7 @@ def test_agent_type(self) -> None: 'agenttype': parse_column_options('agenttype'), 'lastname': parse_column_options('lastname'), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -545,6 +551,7 @@ def test_wbcols_with_ignoreWhenBlank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -570,6 +577,7 @@ def test_wbcols_with_ignoreWhenBlank_and_default(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -601,6 +609,7 @@ def test_wbcols_with_ignoreNever(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -625,6 +634,7 @@ def test_wbcols_with_ignoreAlways(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreAlways", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -652,6 +662,7 @@ def test_wbcols_with_default(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -679,6 +690,7 @@ def test_wbcols_with_default_matching(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -709,6 +721,7 @@ def test_wbcols_with_default_and_null_disallowed(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -737,6 +750,7 @@ def test_wbcols_with_default_blank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default=""), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -766,6 +780,7 @@ def test_wbcols_with_null_disallowed(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -790,6 +805,7 @@ def test_wbcols_with_null_disallowed_and_ignoreWhenBlank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -818,6 +834,7 @@ def test_wbcols_with_null_disallowed_and_ignoreAlways(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreAlways", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} diff --git a/specifyweb/workbench/upload/tests/testschema.py b/specifyweb/workbench/upload/tests/testschema.py index 695a5803654..1a0aa8080e8 100644 --- a/specifyweb/workbench/upload/tests/testschema.py +++ b/specifyweb/workbench/upload/tests/testschema.py @@ -66,7 +66,7 @@ class OtherSchemaTests(unittest.TestCase): @given(name=infer, wbcols=infer) def test_validate_upload_table_to_json(self, name: str, wbcols: Dict[str, ColumnOptions]): - upload_table = UploadTable(name=name, wbcols=wbcols, static={}, toOne={}, toMany={}) + upload_table = UploadTable(name=name, wbcols=wbcols, overrideScope=None, static={}, toOne={}, toMany={}) validate(upload_table.unparse(), schema) @given(column_opts=from_schema(schema['definitions']['columnOptions'])) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index 07b2cf5a56f..72c85faa7c2 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -1,13 +1,73 @@ - from ..upload_plan_schema import schema, parse_plan -from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable +from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable, DeferredScopeUploadTable, ColumnOptions, ExtendedColumnOptions +from ..upload import do_upload -from .base import UploadTestsBase +from specifyweb.specify import models +from .base import UploadTestsBase, get_table from . import example_plan class ScopingTests(UploadTestsBase): + def setUp(self) -> None: + super().setUp() + self.rel_type_name = "ToRightSide" + + self.right_side_collection = get_table('Collection').objects.create( + catalognumformatname='test', + collectionname='RightSideTest', + isembeddedcollectingevent=False, + discipline=self.discipline) + + get_table('Collectionreltype').objects.create( + name = self.rel_type_name, + leftsidecollection = self.collection, + rightsidecollection = self.right_side_collection, + ) + + self.collection_rel_plan = { + "baseTableName": "collectionrelationship", + "uploadable": { + "uploadTable": { + "wbcols": {}, + "static": {}, + "toOne": { + "leftside": { + "uploadTable": { + "wbcols": { + "catalognumber": "Cat #" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + }, + "rightside": { + "uploadTable": { + "wbcols": { + "catalognumber": "Cat # (2)" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + }, + "collectionreltype": { + "uploadTable": { + "wbcols": { + "name": "Collection Rel Type" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + } + }, + "toMany": {} + } + } + } + def test_embedded_collectingevent(self) -> None: self.collection.isembeddedcollectingevent = True self.collection.save() @@ -47,3 +107,115 @@ def test_embedded_paleocontext_in_collectionobject(self) -> None: ).apply_scoping(self.collection) self.assertIsInstance(plan.toOne['paleocontext'], ScopedOneToOneTable) + + def collection_rel_type_being_deferred(self) -> None: + + parsed_plan = parse_plan(self.collection, self.collection_rel_plan) + + expected_plan = UploadTable( + name='Collectionrelationship', + wbcols={}, + static={}, + toOne={ + 'leftside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='leftsidecollection', + filter_field='name', + overrideScope=None + ), + 'rightside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat # (2)', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='rightsidecollection', + filter_field='name', + overrideScope=None + ), + 'collectionreltype': UploadTable( + name='Collectionreltype', + wbcols={'name': ColumnOptions(column='Collection Rel Type', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + overrideScope=None + ) + }, + toMany={}, + overrideScope=None) + + self.assertEqual(parsed_plan, expected_plan) + + def deferred_scope_table_ignored_when_scoping_applied(self): + scoped_upload_plan = parse_plan(self.collection_rel_plan).apply_scoping(self.collection) + + expected_scoping = ScopedUploadTable( + name='Collectionrelationship', + wbcols={}, + static={}, + toOne={ + 'leftside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='leftsidecollection', + filter_field='name', + overrideScope=None), + 'rightside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat # (2)', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='rightsidecollection', + filter_field='name', + overrideScope=None), + 'collectionreltype': ScopedUploadTable( + name='Collectionreltype', + wbcols={'name': ExtendedColumnOptions( + column='Collection Rel Type', + matchBehavior='ignoreNever', + nullAllowed=True, + default=None, + uiformatter=None, + schemaitem= models.Splocalecontaineritem.objects.get(name='name', container=models.Splocalecontainer.objects.get(name='collectionreltype', discipline_id=self.discipline.id)), + picklist=None, + dateformat='%m/%d/%Y')}, + static={}, + toOne={}, + toMany={}, + scopingAttrs={}, + disambiguation=None)}, + + toMany={}, + scopingAttrs={}, + disambiguation=None) + + self.assertEqual(scoped_upload_plan, expected_scoping) + + def collection_rel_uploaded_in_correct_collection(self): + scoped_plan = parse_plan(self.collection_rel_plan).apply_scoping(self.collection) + rows = [ + {'Collection Rel Type': self.rel_type_name, 'Cat # (2)': '999', 'Cat #': '23'}, + {'Collection Rel Type': self.rel_type_name, 'Cat # (2)': '888', 'Cat #': '32'} + ] + do_upload(self.collection, rows, scoped_plan, self.agent.id) + left_side_cat_nums = [n.zfill(9) for n in '32 23'.split()] + right_side_cat_nums = [n.zfill(9) for n in '999 888'.split()] + + left_side_query = models.Collectionobject.objects.filter(collection_id=self.collection.id, catalognumber__in=left_side_cat_nums) + right_side_query = models.Collectionobject.objects.filter(collection_id=self.right_side_collection.id, catalognumber__in=right_side_cat_nums) + + self.assertEqual(left_side_query.count(), 2) + self.assertEqual(right_side_query.count(), 2) diff --git a/specifyweb/workbench/upload/tests/testuploading.py b/specifyweb/workbench/upload/tests/testuploading.py index f24dbf4fff4..9c1da62a454 100644 --- a/specifyweb/workbench/upload/tests/testuploading.py +++ b/specifyweb/workbench/upload/tests/testuploading.py @@ -328,11 +328,13 @@ def test_attachmentimageattribute(self) -> None: plan = UploadTable( name='Attachment', wbcols={'guid': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={'attachmentimageattribute': UploadTable( name='Attachmentimageattribute', wbcols={'height': parse_column_options('height')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -354,11 +356,13 @@ def test_collectingtripattribute(self) -> None: plan = UploadTable( name='Collectingtrip', wbcols={'collectingtripname': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingtripattribute': UploadTable( name='Collectingtripattribute', wbcols={'integer1': parse_column_options('integer')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -380,12 +384,14 @@ def test_preparationattribute(self) -> None: plan = UploadTable( name='Preparation', wbcols={'guid': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={ 'preptype': UploadTable( name='Preptype', wbcols={'name': parse_column_options('preptype')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -393,6 +399,7 @@ def test_preparationattribute(self) -> None: 'preparationattribute': UploadTable( name='Preparationattribute', wbcols={'number1': parse_column_options('integer')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -400,6 +407,7 @@ def test_preparationattribute(self) -> None: 'collectionobject': UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -425,11 +433,13 @@ def test_collectionobjectattribute(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toMany={}, toOne={'collectionobjectattribute': UploadTable( name='Collectionobjectattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -453,11 +463,13 @@ def test_collectingeventattribute(self) -> None: plan = UploadTable( name='Collectingevent', wbcols={'stationfieldnumber': parse_column_options('sfn')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingeventattribute': UploadTable( name='Collectingeventattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -534,6 +546,7 @@ def test_ambiguous_one_to_one_match(self) -> None: plan = UploadTable( name='Collectingevent', wbcols={'stationfieldnumber': parse_column_options('sfn')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingeventattribute': UploadTable( @@ -556,11 +569,13 @@ def test_null_record_with_ambiguous_one_to_one(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toMany={}, toOne={'collectionobjectattribute': UploadTable( name='Collectionobjectattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -667,6 +682,7 @@ def test_ordernumber(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -677,6 +693,7 @@ def test_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author1')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -688,6 +705,7 @@ def test_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author2')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -708,6 +726,7 @@ def test_no_override_ordernumber(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -718,6 +737,7 @@ def test_no_override_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author1')}, + overrideScope=None, static={}, toOne={}, toMany={} diff --git a/specifyweb/workbench/upload/tomany.py b/specifyweb/workbench/upload/tomany.py index 36b0bcc3c03..7b08ef53180 100644 --- a/specifyweb/workbench/upload/tomany.py +++ b/specifyweb/workbench/upload/tomany.py @@ -56,12 +56,12 @@ def disambiguate(self, disambiguation: Disambiguation) -> "ScopedToManyRecord": def get_treedefs(self) -> Set: return set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]) -> Union["BoundToManyRecord", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict], row_index: Optional[int] = None) -> Union["BoundToManyRecord", ParseFailures]: parsedFields, parseFails = parse_many(collection, self.name, self.wbcols, row) toOne: Dict[str, BoundUploadable] = {} for fieldname, uploadable in self.toOne.items(): - result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache) + result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result, ParseFailures): parseFails += result.failures else: diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 660ed31ae9b..14fc286bdab 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -57,7 +57,7 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": def get_treedefs(self) -> Set: return set([self.treedef]) - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundTreeRecord", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: parsedFields: Dict[str, List[ParseResult]] = {} parseFails: List[ParseFailure] = [] for rank, cols in self.ranks.items(): @@ -94,8 +94,8 @@ def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": return ScopedMustMatchTreeRecord(*s) class ScopedMustMatchTreeRecord(ScopedTreeRecord): - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundMustMatchTreeRecord", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundMustMatchTreeRecord", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return b if isinstance(b, ParseFailures) else BoundMustMatchTreeRecord(*b) class TreeDefItemWithParseResults(NamedTuple): diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 0a9bf6e2b82..d16b63065b6 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -11,9 +11,11 @@ from jsonschema import validate # type: ignore from specifyweb.specify import models +from specifyweb.specify.datamodel import datamodel from specifyweb.specify.auditlog import auditlog from specifyweb.specify.datamodel import Table from specifyweb.specify.tree_extras import renumber_tree, reset_fullnames +from specifyweb.workbench.upload.upload_table import DeferredScopeUploadTable, ScopedUploadTable from . import disambiguation from .upload_plan_schema import schema, parse_plan_with_basetable from .upload_result import Uploaded, UploadResult, ParseFailures, \ @@ -187,6 +189,38 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab base_table, plan = parse_plan_with_basetable(collection, plan) return base_table, plan.apply_scoping(collection) +def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: + + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int): # -> models.Collection + # to call this function, we always know upload_plan is either a DeferredScopeUploadTable or ScopedUploadTable + related_uploadable: Union[ScopedUploadTable, DeferredScopeUploadTable] = upload_plan.toOne[deferred_upload_plan.related_key] # type: ignore + related_column_name = related_uploadable.wbcols['name'][0] + filter_value = rows[row_index][related_column_name] # type: ignore + + filter_search = {deferred_upload_plan.filter_field : filter_value} + + related_table = datamodel.get_table(deferred_upload_plan.related_key) + if related_table is not None: + related = getattr(models, related_table.django_name).objects.get(**filter_search) + collection_id = getattr(related, deferred_upload_plan.relationship_name).id + collection = getattr(models, "Collection").objects.get(id=collection_id) + return collection + + if hasattr(upload_plan, 'toOne'): + # Without type ignores, MyPy throws the following error: "ScopedUploadable" has no attribute "toOne" + # MyPy expects upload_plan to be of type ScopedUploadable (from the paramater type) + # but within this if-statement we know that upload_plan is always an UploadTable + # (or more specifically, one if its derivatives: DeferredScopeUploadTable or ScopedUploadTable) + + for key, uploadable in upload_plan.toOne.items(): # type: ignore + _uploadable = uploadable + if hasattr(_uploadable, 'toOne'): _uploadable = apply_deferred_scopes(_uploadable, rows) + if isinstance(_uploadable, DeferredScopeUploadTable): + _uploadable = _uploadable.add_colleciton_override(collection_override_function) + upload_plan.toOne[key] = _uploadable # type: ignore + + return upload_plan + def do_upload( collection, @@ -201,6 +235,7 @@ def do_upload( cache: Dict = {} _auditor = Auditor(collection=collection, audit_log=None if no_commit else auditlog) total = len(rows) if isinstance(rows, Sized) else None + deffered_upload_plan = apply_deferred_scopes(upload_plan, rows) with savepoint("main upload"): tic = time.perf_counter() results: List[UploadResult] = [] @@ -208,7 +243,7 @@ def do_upload( _cache = cache.copy() if cache is not None and allow_partial else cache da = disambiguations[i] if disambiguations else None with savepoint("row upload") if allow_partial else no_savepoint(): - bind_result = upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache) + bind_result = deffered_upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache, i) result = UploadResult(bind_result, {}, {}) if isinstance(bind_result, ParseFailures) else bind_result.process_row() results.append(result) if progress is not None: @@ -224,7 +259,7 @@ def do_upload( if no_commit: raise Rollback("no_commit option") else: - fixup_trees(upload_plan, results) + fixup_trees(deffered_upload_plan, results) return results diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 1f05b322828..3826fbf4263 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -4,11 +4,12 @@ from specifyweb.specify.load_datamodel import DoesNotExistError from specifyweb.specify import models -from .upload_table import UploadTable, OneToOneTable, MustMatchTable +from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord from .treerecord import TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions +from .scoping import DEFERRED_SCOPING schema: Dict = { @@ -42,6 +43,14 @@ 'type': 'object', 'description': 'The uploadTable structure defines how to upload data for a given table.', 'properties': { + 'overrideScope' : { + 'description' : '', + 'type' : 'object', + 'properties': { + 'collection' : { '$ref': '#/definitions/scopingOverride'} + }, + 'additionalProperties' : False, + }, 'wbcols': { '$ref': '#/definitions/wbcols' }, 'static': { '$ref': '#/definitions/static' }, 'toOne': { '$ref': '#/definitions/toOne' }, @@ -196,6 +205,12 @@ {'ispublic': True, 'license': 'CC BY-NC-ND 2.0'} ] }, + 'scopingOverride' : { + 'description' : '', + 'default' : None, + 'oneOf' : [ {'type': 'integer'}, + {'type': 'null'}] + } } } @@ -229,14 +244,42 @@ def parse_upload_table(collection, table: Table, to_parse: Dict) -> UploadTable: def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) + + def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, deferred_information: Tuple[str, str]) -> DeferredScopeUploadTable: + related_key = DEFERRED_SCOPING[deferred_information][0] + filter_field = DEFERRED_SCOPING[deferred_information][1] + relationship_name = DEFERRED_SCOPING[deferred_information][2] + + return DeferredScopeUploadTable( + name=table.django_name, + related_key=related_key, + relationship_name=relationship_name, + filter_field=filter_field, + overrideScope= to_parse['overrideScope'] if 'overrideScope' in to_parse.keys() else None, + wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, + static=to_parse['static'], + toOne={ + key: defer_scope_upload_table(collection, rel_table(key), to_one, (table.django_name, key)) + if (table.django_name, key) in DEFERRED_SCOPING.keys() + else parse_uploadable(collection, rel_table(key), to_one) + for key, to_one in to_parse['toOne'].items() + }, + toMany={ + key: [parse_to_many_record(default_collection, rel_table(key), record) for record in to_manys] + for key, to_manys in to_parse['toMany'].items() + } + ) return UploadTable( name=table.django_name, + overrideScope= to_parse['overrideScope'] if 'overrideScope' in to_parse.keys() else None, wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ - key: parse_uploadable(collection, rel_table(key), to_one) - for key, to_one in to_parse['toOne'].items() + key: defer_scope_upload_table(collection, rel_table(key), to_one['uploadTable'], (table.django_name, key)) + if (table.django_name, key) in DEFERRED_SCOPING.keys() + else parse_uploadable(collection, rel_table(key), to_one) + for key, to_one in to_parse['toOne'].items() }, toMany={ key: [parse_to_many_record(collection, rel_table(key), record) for record in to_manys] diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 0d20aede145..f299853b3f5 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal, cast from django.db import transaction, IntegrityError @@ -26,15 +26,161 @@ class UploadTable(NamedTuple): toOne: Dict[str, Uploadable] toMany: Dict[str, List[ToManyRecord]] + overrideScope: Optional[Dict[Literal['collection'], Optional[int]]] = None + def apply_scoping(self, collection) -> "ScopedUploadTable": - from .scoping import apply_scoping_to_uploadtable as apply_scoping - return apply_scoping(self, collection) + from .scoping import apply_scoping_to_uploadtable + return apply_scoping_to_uploadtable(self, collection) + + def get_cols(self) -> Set[str]: + return set(cd.column for cd in self.wbcols.values()) \ + | set(col for u in self.toOne.values() for col in u.get_cols()) \ + | set(col for rs in self.toMany.values() for r in rs for col in r.get_cols()) + + def _to_json(self) -> Dict: + result = dict( + wbcols={k: v.to_json() for k,v in self.wbcols.items()}, + static=self.static + ) + result['toOne'] = { + key: uploadable.to_json() + for key, uploadable in self.toOne.items() + } + result['toMany'] = { + key: [to_many.to_json() for to_many in to_manys] + for key, to_manys in self.toMany.items() + } + return result + + def to_json(self) -> Dict: + return { 'uploadTable': self._to_json() } + + def unparse(self) -> Dict: + return { 'baseTableName': self.name, 'uploadable': self.to_json() } + +class DeferredScopeUploadTable(NamedTuple): + ''' In the case that the scoping of a record in a WorkBench upload can not be known + until the values of the rows are known, the scope of the record should be deferred until + the row is being processed. + + When the upload table is parsed in .upload_plan_schema.py, if a table contains a field which scoping + is unknown, a DeferredScope UploadTable is created. + + As suggested by the name, the DeferredScope UploadTable is not scoped or disambiguated until its bind() + method is called. In which case, the rows of the dataset are known and the scoping can be deduced + ''' + name: str + wbcols: Dict[str, ColumnOptions] + static: Dict[str, Any] + toOne: Dict[str, Uploadable] + toMany: Dict[str, List[ToManyRecord]] + + related_key: str + relationship_name: str + filter_field: str + + disambiguation: Disambiguation = None + + """ In a DeferredScopeUploadTable, the overrideScope value can be either an integer + (which follows the same logic as in UploadTable), or a function which has the parameter + signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection + (see apply_deferred_scopes in .upload.py) + + overrideScope should be of type + Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] + + But recursively using the type within the class definition of a NamedTuple is not supported in our version + of mypy + See https://github.com/python/mypy/issues/8695 + """ + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[[Any, int], Any]]]] = None + + + # Typehint for return type should be: Union["ScopedUploadTable", "DeferredScopeUploadTable"] + def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", Any]: + if not defer: + from .scoping import apply_scoping_to_uploadtable + return apply_scoping_to_uploadtable(self, collection) + else: return self def get_cols(self) -> Set[str]: return set(cd.column for cd in self.wbcols.values()) \ | set(col for u in self.toOne.values() for col in u.get_cols()) \ | set(col for rs in self.toMany.values() for r in rs for col in r.get_cols()) + + """ + The Typehint for parameter collection should be: Union[int, Callable[["DeferredScopeUploadTable", int], Any]] + The Typehint for return type should be: "DeferredScopeUploadTable" + """ + def add_colleciton_override(self, collection: Union[int, Callable[[Any, int], Any]]) -> Any: + ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override + To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), + which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. + + Note that _replace(**kwargs) does not modify the original object. It insteads creates a new object with the same attributes except for + those added/changed in the paramater kwargs. + + ''' + return self._replace(overrideScope = {"collection": collection}) + + def disambiguate(self, da: Disambiguation): + '''Disambiguation should only be used when the UploadTable is completely Scoped. + + When a caller attempts to disambiguate a DeferredScope UploadTable, create and return + a copy of the DeferredScope Upload Table with the Disambiguation stored in a + 'disambiguation' attribute. + + If this attribute exists when the DeferredScoped UploadTable is scoped, + then disambiguate the new Scoped UploadTable using the stored Disambiguation + ''' + return self._replace(disambiguation = da) + + def get_treedefs(self) -> Set: + """ This method is needed because a ScopedUploadTable may call this when calling its own get_treedefs() + This returns an empty set unless the toOne or toMany Uploadable is a TreeRecord + """ + return ( + set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) | # type: ignore + set(td for toMany in self.toMany.values() for tmr in toMany for td in tmr.get_treedefs()) # type: ignore + ) + + def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundUploadTable", ParseFailures]: + + scoped = None + + ''' If the collection should be overridden and an integer (collection id) is provided, + then get the collection with that id and apply the proper scoping. + + Otherwise, if a funciton is provided (see apply_deferred_scopes in .upload.py), then call the function + with the row and row_index to get the needed collection + ''' + if self.overrideScope is not None and'collection' in self.overrideScope.keys(): + if isinstance(self.overrideScope['collection'], int): + collection_id = self.overrideScope['collection'] + collection = getattr(models, "Collection").objects.get(id=collection_id) + scoped = self.apply_scoping(collection, defer=False) + elif callable(self.overrideScope['collection']): + collection = self.overrideScope['collection'](self, row_index) if row_index is not None else default_collection + scoped = self.apply_scoping(collection, defer=False) + + # If the collection/scope should not be overriden, defer to the default behavior and assume + # the record should be uploaded in the logged-in collection + if scoped is None: scoped = self.apply_scoping(default_collection, defer=False) + + # self.apply_scoping is annotated to Union["ScopedUploadTable", Any] + # But at this point we know the variable scoped will always be a ScopedUploadTable + # We tell typing the type of the variable scoped will be ScopedUploadTable with the cast() function + scoped = cast(ScopedUploadTable, scoped) + + # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new + # ScopedUploadTable + # Because ScopedUploadTable.disambiguate() has return type of ScopedUploadable, we must specify the type as ScopedUploadTable + scoped_disambiguated = cast(ScopedUploadTable, scoped.disambiguate(self.disambiguation)) if self.disambiguation is not None else scoped + # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures + return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) + def _to_json(self) -> Dict: result = dict( wbcols={k: v.to_json() for k,v in self.wbcols.items()}, @@ -56,6 +202,7 @@ def to_json(self) -> Dict: def unparse(self) -> Dict: return { 'baseTableName': self.name, 'uploadable': self.to_json() } + class ScopedUploadTable(NamedTuple): name: str wbcols: Dict[str, ExtendedColumnOptions] @@ -91,12 +238,13 @@ def get_treedefs(self) -> Set: ) - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundUploadTable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundUploadTable", ParseFailures]: parsedFields, parseFails = parse_many(collection, self.name, self.wbcols, row) toOne: Dict[str, BoundUploadable] = {} for fieldname, uploadable in self.toOne.items(): - result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache) + result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result, ParseFailures): parseFails += result.failures else: @@ -106,7 +254,7 @@ def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, ca for fieldname, records in self.toMany.items(): boundRecords: List[BoundToManyRecord] = [] for record in records: - result_ = record.bind(collection, row, uploadingAgentId, auditor, cache) + result_ = record.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result_, ParseFailures): parseFails += result_.failures else: @@ -138,8 +286,9 @@ def to_json(self) -> Dict: return { 'oneToOneTable': self._to_json() } class ScopedOneToOneTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundOneToOneTable", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundOneToOneTable", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundOneToOneTable(*b) if isinstance(b, BoundUploadTable) else b class MustMatchTable(UploadTable): @@ -151,8 +300,9 @@ def to_json(self) -> Dict: return { 'mustMatchTable': self._to_json() } class ScopedMustMatchTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundMustMatchTable", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundMustMatchTable", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundMustMatchTable(*b) if isinstance(b, BoundUploadTable) else b diff --git a/specifyweb/workbench/upload/uploadable.py b/specifyweb/workbench/upload/uploadable.py index 4e20f563ba2..17045aa6013 100644 --- a/specifyweb/workbench/upload/uploadable.py +++ b/specifyweb/workbench/upload/uploadable.py @@ -39,7 +39,7 @@ class ScopedUploadable(Protocol): def disambiguate(self, disambiguation: Disambiguation) -> "ScopedUploadable": ... - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundUploadable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadable", ParseFailures]: ... def get_treedefs(self) -> Set: