Skip to content

add(staff-association): new model and transformation rules #391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zubeydecivelek wants to merge 2 commits into
base: master
Choose a base branch
from
Open

add(staff-association): new model and transformation rules #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce91c1c
add(staff-association): new model and transformation rules
zubeydecivelek Mar 19, 2026
36dcede
add(weekly-bulletin-drafts): New model and stream
sakshamarora1 Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/bulletin_drafts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2026 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM Bulletin Drafts model."""

from cds_migrator_kit.rdm.records.transform.models.bulletin_issue import (
bull_issue_model,
)
from cds_migrator_kit.rdm.records.transform.models.staff_association import (
staff_association_model,
)
from cds_migrator_kit.transform.overdo import CdsOverdo


class BulletinDraftsModel(CdsOverdo):
"""Translation model for Bulletin Drafts."""

__query__ = """(
980__:"BULLETINSTAFFDRAFT" OR
980__:"BULLETINNEWSDRAFT" OR
980__:"BULLETINOFFICIALDRAFT" OR
980__:"BULLETINTRAININGDRAFT" OR
980__:"BULLETINANNOUNCEDRAFT" OR
980__:"BULLETINEVENTSDRAFT"
)
"""

# Copy-pasted from bulletin issue
__ignore_keys__ = {
"0248_a",
"0248_p",
"0248_q",
"100__m", # email of contributor
"110__a", # corporate author, always CERN, safe to ignore
"300__a", # number of pages
"336__a", # DM metadata
"5831_2", # DM tags 1054836
"5831_5", # DM tags
"5831_a", # DM tags
"5831_c", # DM tags
"5831_f", # DM tags
"5831_i", # DM tags
"5831_k", # DM tags
"5831_u", # DM tags
"5831_3", # DM tags
"5831_6", # DM tags
"5831_n", # DM tags
"5831_b", # DM tags
"5831_o", # DM tags
"583__a", # DM tags
"583__c", # DM tags
"583__z", # DM tags
"594__a", # values: "no", "pub"
"650172", # scheme of subjects
"6531_9", # scheme of keywords
"691__a", # draft/online values, redundant
"700__m", # email of contributor
"773__p", # title of the "CERN Bulletin" series
"773__t", # CERN Bulletin value, redundant
"773__y", # year, duplicate of 260
"8560_f", # contact email
"8564_8", # file id
"8564_s", # bibdoc id
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump
"8564_2", # DM metadata
"8564_q", # DM metadata
"8564_w", # DM metadata
"8564_z", # DM metadata
"8567_2", # DM tags
"8567_q", # DM tags
"8567_w", # DM tags
"8567_d", # DM tags
"906__m", # edit rights, will be granted by the community
"937__c", # last modified by
"937__s", # last modification date
"960__a", # base number
"961__a", # Curation Auditing tag
"961__b", # Curation Auditing tag
"961__c", # Curation Auditing tag
"961__h", # Curation Auditing tag
"961__l", # Curation Auditing tag
"961__x", # Curation Auditing tag
"981__a", # duplicate record id
# "246_1a",
# "690C_a",
}

_default_fields = {
"custom_fields": {"journal:journal": {"title": "CERN Bulletin"}},
"creators": [{"person_or_org": {"type": "organizational", "name": "CERN"}}],
}


bulletin_drafts_model = BulletinDraftsModel(
bases=(staff_association_model, bull_issue_model,),
entry_point_group="cds_migrator_kit.migrator.rules.bulletin_drafts",
)
31 changes: 20 additions & 11 deletions cds_migrator_kit/rdm/records/transform/models/bulletin_issue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,26 @@
class BulletinIssueModel(CdsOverdo):
"""Translation model for Bulletin Issue."""

__query__ = """980__:CERN_BULLETIN_ISSUE OR
980__:CERN_BULLETIN_ARTICLE OR
980__:BULLETINGENERAL OR
980__:BULLETINEVENTS OR
980__:BULLETINANNOUNCE OR
980__:BULLETINBREAKING OR
980__:BULLETINNEWS OR
980__:BULLETINOFFICIAL OR
980__:BULLETINPENSION OR
980__:BULLETINTRAINING OR
980__:BULLETINSOCIAL"""
__query__ = """(
980__:CERN_BULLETIN_ISSUE OR
980__:CERN_BULLETIN_ARTICLE OR
980__:BULLETINGENERAL OR
980__:BULLETINEVENTS OR
980__:BULLETINANNOUNCE OR
980__:BULLETINBREAKING OR
980__:BULLETINNEWS OR
980__:BULLETINOFFICIAL OR
980__:BULLETINPENSION OR
980__:BULLETINTRAINING OR
980__:BULLETINSOCIAL
)
-980__:BULLETINSTAFFDRAFT
-980__:BULLETINNEWSDRAFT
-980__:BULLETINOFFICIALDRAFT
-980__:BULLETINTRAININGDRAFT
-980__:BULLETINANNOUNCEDRAFT
-980__:BULLETINEVENTSDRAFT
"""

__ignore_keys__ = {
"0248_a",
Expand Down
101 changes: 101 additions & 0 deletions cds_migrator_kit/rdm/records/transform/models/staff_association.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2026 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM Staff Association model."""
from cds_migrator_kit.rdm.records.transform.models.bulletin_issue import (
bull_issue_model,
)
from cds_migrator_kit.transform.overdo import CdsOverdo


class StaffAssociationModel(CdsOverdo):
"""Translation model for Staff Association."""

__query__ = """
(
980__:"BULLETINSTAFF"
-980__:CERN_BULLETIN_ARTICLE
-980__:CERN_BULLETIN_ISSUE
)
OR
(
980__:STAFFASSOCIATION
594__:PUB
)
"""

# Copy-pasted from bulletin issue
__ignore_keys__ = {
"0248_a",
"0248_p",
"0248_q",
"100__m", # email of contributor
"110__a", # corporate author, always CERN, safe to ignore
"300__a", # number of pages
"336__a", # DM metadata
"5831_2", # DM tags 1054836
"5831_5", # DM tags
"5831_a", # DM tags
"5831_c", # DM tags
"5831_f", # DM tags
"5831_i", # DM tags
"5831_k", # DM tags
"5831_u", # DM tags
"5831_3", # DM tags
"5831_6", # DM tags
"5831_n", # DM tags
"5831_b", # DM tags
"5831_o", # DM tags
"583__a", # DM tags
"583__c", # DM tags
"583__z", # DM tags
"594__a", # values: "no", "pub"
"650172", # scheme of subjects
"6531_9", # scheme of keywords
"691__a", # draft/online values, redundant
"700__m", # email of contributor
"773__p", # title of the "CERN Bulletin" series
"773__t", # CERN Bulletin value, redundant
"773__y", # year, duplicate of 260
"8560_f", # contact email
"8564_8", # file id
"8564_s", # bibdoc id
"8564_x", # icon thumbnails sizes
"8564_y", # file description - done by files dump
"8564_2", # DM metadata
"8564_q", # DM metadata
"8564_w", # DM metadata
"8564_z", # DM metadata
"8567_2", # DM tags
"8567_q", # DM tags
"8567_w", # DM tags
"8567_d", # DM tags
"906__m", # edit rights, will be granted by the community
"937__c", # last modified by
"937__s", # last modification date
"960__a", # base number
"961__a", # Curation Auditing tag
"961__b", # Curation Auditing tag
"961__c", # Curation Auditing tag
"961__h", # Curation Auditing tag
"961__l", # Curation Auditing tag
"961__x", # Curation Auditing tag
"981__a", # duplicate record id
# "246_1a",
# "690C_a",
}

_default_fields = {
"custom_fields": {"journal:journal": {"title": "CERN Bulletin"}},
Comment thread
sakshamarora1 marked this conversation as resolved.
"creators": [{"person_or_org": {"type": "organizational", "name": "CERN"}}],
}


staff_association_model = StaffAssociationModel(
bases=(bull_issue_model,),
entry_point_group="cds_migrator_kit.migrator.rules.staff_association",
)
15 changes: 8 additions & 7 deletions cds_migrator_kit/rdm/records/transform/transform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,13 +246,14 @@ def _owner(self, json_entry):
user = User.query.filter_by(email=email).one()
return user.id
except NoResultFound:
raise UnexpectedValue(
message=f"{email} not found - did you run user migration?",
stage="transform",
recid=json_entry["legacy_recid"],
value=email,
priority="critical",
)
# return UnexpectedValue(
# message=f"{email} not found - did you run user migration?",
# stage="transform",
# recid=json_entry["legacy_recid"],
# value=email,
# priority="critical",
# )
return "system"

def _match_affiliation(self, affiliation_name, json_entry):
"""Match an affiliation against `CDSMigrationAffiliationMapping` db table."""
Expand Down
20 changes: 19 additions & 1 deletion cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -819,11 +819,29 @@ def related_identifiers_787(self, key, value):
"article":{
"relation_type": {"id": "references"},
"resource_type": {"id": "publication-article"},
}
},
"corresponding video": {
"relation_type": {"id": "references"},
"resource_type": {"id": "audio"},
Comment thread
zubeydecivelek marked this conversation as resolved.
},
Comment thread
zubeydecivelek marked this conversation as resolved.
"bulletin article": {
"relation_type": {"id": "references"},
"resource_type": {"id": "publication-periodicalarticle"},
},
"report": {
"relation_type": {"id": "isderivedfrom"},
"resource_type": {"id": "publication-report"},
},
}

if recid:
if description:
if description not in relation_map.keys():
raise UnexpectedValue(
f"Unexpected relation description {description}",
field=key,
value=value,
)
new_id = {
"identifier": recid,
"scheme": "cds",
Expand Down
29 changes: 29 additions & 0 deletions cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_drafts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2026 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""CDS-RDM Bulletin Drafts rules."""

from cds_migrator_kit.errors import UnexpectedValue

from ...models.bulletin_drafts import bulletin_drafts_model as model

@model.over("resource_type", "^980__", override=True)
def resource_type(self, key, value):
"""Translates resource_type for bulletin drafts."""
value = value.get("a", "").lower()
if value in [
"bulletinstaffdraft",
"bulletinnewsdraft",
"bulletinofficialdraft",
"bulletintrainingdraft",
"bulletinannouncedraft",
"bulletineventsdraft",
]:
return {"id": "publication-periodicalarticle"}
raise UnexpectedValue(
"Unknown resource type (BULLETIN DRAFTS)", field=key, value=value
)
2 changes: 1 addition & 1 deletion cds_migrator_kit/rdm/records/transform/xml_processing/rules/bulletin_issue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def urls_bulletin_bis(self, key, value):


@model.over("custom_fields_journal", "(^916__)", override=True)
def issue_number(self, key, value):
def custom_fields_journal(self, key, value):
_custom_fields = self.get("custom_fields", {})

issue = value.get("z")
Expand Down
Loading
Loading