From 9b19985eb7ea388246c31063d49a73321b2d0d33 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 22 May 2024 16:07:43 -0400
Subject: [PATCH 001/182] feat: support package mapping IPM everywhere
Even %SYS, but not using the %ALL namespace because that ends up overriding namespace-specific mappings (gah).
---
preload/cls/IPM/Installer.cls | 14 ++-
src/cls/IPM/Main.cls | 114 ++++++++++++++++--
src/cls/IPM/Utils/Migration.cls | 9 +-
.../Test/PM/Migration/GloballyEnabled.cls | 10 ++
4 files changed, 133 insertions(+), 14 deletions(-)
create mode 100644 tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/GloballyEnabled.cls
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 0a496183..5a5a7200 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -35,6 +35,7 @@ XData PM [ XMLNamespace = INSTALLER ]
+
@@ -43,10 +44,10 @@ XData PM [ XMLNamespace = INSTALLER ]
XData MapOnly [ XMLNamespace = INSTALLER ]
{
-
+
-
+
@@ -79,7 +80,6 @@ ClassMethod Map(ByRef pVars, pLogLevel As %Integer, pInstaller As %Installer.Ins
Do %code.WriteLine(" Set sc = ##class(Config.Namespaces).Get(prevNS, .properties)")
Do %code.WriteLine(" Set pVars(""CURNSRoutineDB"")=$Get(properties(""Routines""))")
Do %code.WriteLine(" Set pVars(""CURNSGlobalDB"")=$Get(properties(""Globals""))")
- Do %code.WriteLine(" If ##class(Config.MapPackages).Exists(prevNS,""%IPM"") { Quit $$$OK }")
Quit ##class(%Installer.Manifest).%Generate(%compiledclass, %code, "MapOnly")
}
@@ -115,6 +115,14 @@ ClassMethod ZPMCompile()
Quit ##class(%IPM.Main).Shell("ZPM compile")
}
+ClassMethod MapIfLegacy()
+{
+ If ##class(%IPM.Utils.Migration).HasLegacyZPMPackage() {
+ Quit ##class(%IPM.Main).Shell("enable -globally -map")
+ }
+ Quit $$$OK
+}
+
ClassMethod EndCompile(qstruct) As %Status
{
#; Behave as simple installer
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 3e8403bc..5c4882dd 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -611,11 +611,16 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
+
+
+
+ enable -map -globally
+
enable -v 0.3.4 -q -ns NS1,NS2,NS3
@@ -1132,8 +1137,9 @@ ClassMethod GetVersionModule(name, namespace = {$namespace})
/// Version client and registry
ClassMethod Version(ByRef pCommandInfo) [ Internal ]
{
- Do ..GetListModules("*", $$$IPMModuleName_","_$$$IPMServerRegistryModuleName, .list)
- Do ..DisplayModules(.list)
+ Do ..GetListModules("*", $$$IPMModuleName_","_$$$IPMServerRegistryModuleName, .list)
+ Do ..AddIPMMappedNamespaces(.list)
+ Do ..DisplayModules(.list)
// Get URL current registry
Set tRes = ##class(%IPM.Repo.Remote.Definition).ExtentFunc()
@@ -1150,6 +1156,38 @@ ClassMethod Version(ByRef pCommandInfo) [ Internal ]
Quit $$$OK
}
+ClassMethod AddIPMMappedNamespaces(ByRef list) [ Private ]
+{
+ Do ..GetListNamespace(.allNS,"*")
+ For i=1:1:+$Get(list) {
+ Set namespace = $ListGet(list(i))
+ If (namespace '= "") {
+ Kill allNS(namespace)
+ }
+ Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(namespace,"%IPM")
+ Set mappedFrom(sourceDB) = namespace
+ Merge mappedFrom(sourceDB,"info") = list(i)
+ }
+
+ Set namespace = ""
+ For {
+ Set namespace = $Order(allNS(namespace))
+ Quit:namespace=""
+
+ Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(namespace,"%IPM")
+ If $Data(mappedFrom(sourceDB),sourceNamespace) {
+ Kill info
+ Merge info = mappedFrom(sourceDB,"info")
+ Set info = $lb(namespace)
+ Set info("modules",1,"Installed In") = sourceNamespace
+ Merge list($i(list)) = info
+ If $Length(namespace) > +$Get(list("width")) {
+ Set list("width") = $Length(namespace)
+ }
+ }
+ }
+}
+
/// @API.Method
ClassMethod IsAvailable(pModuleName As %String, pNamespace As %String = "", pArgs...) As %Boolean
{
@@ -2602,14 +2640,18 @@ ZPM(pArgs...)
ClassMethod EnableIPM(ByRef pCommandInfo)
{
- Write !, "Version of IPM in current namespace: "
- Do ..GetListModules($Namespace,$$$IPMModuleName,.list)
- If $Data(list($$$IPMModuleName)) {
- Write !,($namespace)_"> "_$$$FormattedLine($$$Green,$$$IPMModuleName_" ")_$ListGet(list($$$IPMModuleName),1)
+ New $Namespace
+ Set initNamespace = $Namespace
+ If ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName,.ipmModuleId) {
+ Set modDef = ##class(%IPM.Storage.Module).%OpenId(ipmModuleId,,.sc)
+ $$$ThrowOnError(sc)
+ Write !, "Version of IPM in current namespace: "
+ Write !,($namespace)_"> "_$$$FormattedLine($$$Green,$$$IPMModuleName_" ")_modDef.VersionString
}
- Kill list
Set quiet = $$$HasModifier(pCommandInfo,"quiet")
+ Set preview = $$$HasModifier(pCommandInfo,"preview")
+ Set map = $$$HasModifier(pCommandInfo,"map")
Set globally = $$$HasModifier(pCommandInfo, "globally")
Set localOnly = $$$HasModifier(pCommandInfo, "local-only")
Set version = $$$GetModifier(pCommandInfo, "version")
@@ -2627,6 +2669,59 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
If ((namespaces '= "") && globally) {
$$$ThrowOnError($$$ERROR($$$GeneralError,"Cannot specify namespaces and global installation flag at the same time."))
}
+ If (version '= "") && (globally && map) {
+ $$$ThrowOnError($$$ERROR($$$GeneralError,"Cannot specify version when mapping the currently-installed version globally."))
+ }
+ If map && 'globally && (namespaces = "") {
+ $$$ThrowOnError($$$ERROR($$$GeneralError,"If mapping from the current namespace's routine database with -map, must specify either -globally or a list of namespaces with -ns"))
+ }
+
+ If map {
+ If globally {
+ Set namespaces = ""
+ Do ..GetListNamespace(.list)
+ Kill list("%ALL")
+ Set key = ""
+ For {
+ Set key = $Order(list(key))
+ Quit:key=""
+ Set namespaces = namespaces _ $ListBuild(key)
+ }
+ } Else {
+ Set namespaces = $ListFromString(namespaces)
+ }
+ Set pointer = 0
+ While $ListNext(namespaces,pointer,namespace) {
+ Set $Namespace = namespace
+ If $$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName) {
+ If 'quiet || preview {
+ Write !,"Skipping "_namespace_" - IPM already installed."
+ }
+ Continue
+ }
+ Set $Namespace = initNamespace
+ If preview {
+ Write !,"Would add IPM mappings to "_namespace
+ Continue
+ }
+ If 'quiet {
+ Write !,"Mapping %IPM package in "_namespace_" equivalently to "_initNamespace
+ }
+ $$$ThrowOnError(##class(%IPM.Utils.Build).MapPackageEquivalently("%IPM",initNamespace,namespace))
+ If 'quiet {
+ Write !,"Mapping %IPM.* routines in "_namespace_" equivalently to "_initNamespace
+ }
+ $$$ThrowOnError(##class(%IPM.Utils.Build).MapRoutineEquivalently("%IPM.*",initNamespace,,namespace))
+ }
+ Set $Namespace = initNamespace
+ If preview {
+ Write !,"Preview mode; no configuration changes were made."
+ } ElseIf 'quiet {
+ Write !,"Done.",!
+ Do ..Version()
+ }
+ Return
+ }
New $Namespace
// 1. Get list of local IPM artifacts under //lib/ipm/
@@ -2914,7 +3009,8 @@ ClassMethod DisplayModules(ByRef pList, pNumbered As %Boolean = 0, pWithNamespac
$ListBuild("Author.Person", $$$Yellow, "Author"),
$ListBuild("Origin", $$$Blue),
$ListBuild("AllVersions", $$$Blue, "Versions"),
- $ListBuild("Repository", $$$Blue)
+ $ListBuild("Repository", $$$Blue),
+ $ListBuild("Installed In", $$$Blue)
)
Set width = $Get(pList("width")) + 1
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 9b329785..520f5bb5 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -14,10 +14,15 @@ ClassMethod RunAll(verbose As %Boolean = 1) As %Status
Quit sc
}
-ClassMethod MigrateZPMToIPM(verbose As %Boolean = 1)
+ClassMethod HasLegacyZPMPackage()
{
Set oldTopPackage = "%ZPM.PackageManager."
- If $Order(^oddCOM(oldTopPackage)) '[ oldTopPackage {
+ Quit $Order(^oddCOM(oldTopPackage)) [ oldTopPackage
+}
+
+ClassMethod MigrateZPMToIPM(verbose As %Boolean = 1)
+{
+ If '..HasLegacyZPMPackage() {
Write:verbose !,"Older IPM version not found; nothing to migrate.",!
Quit
}
diff --git a/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/GloballyEnabled.cls b/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/GloballyEnabled.cls
new file mode 100644
index 00000000..b92b8bc2
--- /dev/null
+++ b/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/GloballyEnabled.cls
@@ -0,0 +1,10 @@
+Class Test.PM.Migration.GloballyEnabled Extends %UnitTest.TestCase
+{
+
+Method TestEnabledInSYS()
+{
+ Do $$$AssertTrue($Data(^|"%SYS"|oddCOM("%IPM.Main")))
+ Do $$$AssertEquals(##class(%SYS.Namespace).GetPackageDest($Namespace,"%IPM"),##class(%SYS.Namespace).GetPackageDest("%SYS","%IPM"))
+}
+
+}
From 68d311fde6527650476c7f95ee28ade9d80877f2 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Fri, 31 May 2024 14:38:15 -0400
Subject: [PATCH 002/182] fix: reintroduce support for other types in PKG
Also adds tests, but it seems that this all just works.
---
src/cls/IPM/Storage/ResourceReference.cls | 2 +-
.../Test/PM/Integration/Resources.cls | 11 +++++++++++
.../cls/ResourceTest/APackage/IncludeFile.inc | 7 +++++++
.../inc/ResourceTest/APackage/AnotherIncludeFile.inc | 7 +++++++
.../PM/Integration/_data/resource-test/module.xml | 1 +
5 files changed, 27 insertions(+), 1 deletion(-)
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/resource-test/cls/ResourceTest/APackage/IncludeFile.inc
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/resource-test/inc/ResourceTest/APackage/AnotherIncludeFile.inc
diff --git a/src/cls/IPM/Storage/ResourceReference.cls b/src/cls/IPM/Storage/ResourceReference.cls
index 890f6feb..c24b2477 100644
--- a/src/cls/IPM/Storage/ResourceReference.cls
+++ b/src/cls/IPM/Storage/ResourceReference.cls
@@ -126,7 +126,7 @@ ClassMethod GetChildren(pResourceName As %String, pModuleName As %String, pCheck
// PKG extension should only cover .CLS files
Set tFilesResult = ##class(%SQL.Statement).%ExecDirect(,
"select Name from %Library.RoutineMgr_StudioOpenDialog(?,'',1,1,1,0,0)",
- tPackage_"*.cls")
+ tPackage_"*.cls,"_tPackage_"*.mac,"_tPackage_"*.int,"_tPackage_"*.inc")
If (tFilesResult.%SQLCODE < 0) {
Set tSC = $$$ERROR($$$SQLCode,tFilesResult.%SQLCODE,tFilesResult.%Message)
Quit
diff --git a/tests/integration_tests/Test/PM/Integration/Resources.cls b/tests/integration_tests/Test/PM/Integration/Resources.cls
index f6d4ae4f..c48e2116 100644
--- a/tests/integration_tests/Test/PM/Integration/Resources.cls
+++ b/tests/integration_tests/Test/PM/Integration/Resources.cls
@@ -45,6 +45,17 @@ Method TestResourceTypes()
Do $$$AssertEquals($Get(^ResourceTest),"Hello World","^ResourceTest was imported properly.")
Do $$$AssertEquals($$MyFunction^ResourceTest(),42,"ResourceTest routine exists and was compiled (and, by extension, ResourceTest.inc as well).")
+
+ // Assert that ownership of .INC files is appropriate
+ &sql(select ID into :rrID from %IPM_Storage.ResourceReference where ModuleItem->Name = 'ResourceTest' and Name = 'ResourceTest.APackage.PKG')
+ $$$ThrowSQLIfError(SQLCODE,%message)
+ Set rr = ##class(%IPM.Storage.ResourceReference).%OpenId(rrID,,.sc)
+ $$$ThrowOnError(sc)
+ Do $$$AssertStatusOK(rr.ResolveChildren(.res))
+ If $$$AssertTrue($Data(res("ResourceTest.APackage.IncludeFile.inc"))) {
+ Do $$$AssertEquals(res("ResourceTest.APackage.IncludeFile.inc","RelativePath"),"cls/ResourceTest/APackage/IncludeFile.inc")
+ }
+ Do $$$AssertNotTrue($Data(res("ResourceTest.APackage.AnotherIncludeFile.inc")))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/resource-test/cls/ResourceTest/APackage/IncludeFile.inc b/tests/integration_tests/Test/PM/Integration/_data/resource-test/cls/ResourceTest/APackage/IncludeFile.inc
new file mode 100644
index 00000000..5d551f03
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/resource-test/cls/ResourceTest/APackage/IncludeFile.inc
@@ -0,0 +1,7 @@
+^INC^Save for Source Control^^~Format=Cache.S~^UTF8
+%RO
+ResourceTest.APackage.IncludeFile^INC^^^0
+#define answer 42
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/resource-test/inc/ResourceTest/APackage/AnotherIncludeFile.inc b/tests/integration_tests/Test/PM/Integration/_data/resource-test/inc/ResourceTest/APackage/AnotherIncludeFile.inc
new file mode 100644
index 00000000..96ecbbcf
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/resource-test/inc/ResourceTest/APackage/AnotherIncludeFile.inc
@@ -0,0 +1,7 @@
+^INC^Save for Source Control^^~Format=Cache.S~^UTF8
+%RO
+ResourceTest.APackage.AnotherIncludeFile^INC^^^0
+#define answer 42
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
index 1ec9de1d..59edbcef 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
@@ -31,6 +31,7 @@
+
From 2ce44ac912e2d606004d5434870b2be171fac100 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 12 Jun 2024 20:05:53 -0400
Subject: [PATCH 003/182] fix: exotic resources can still publish deployed
If we have unit tests, exotic document types, etc., we won't hit now.
---
src/cls/IPM/Lifecycle/Module.cls | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Lifecycle/Module.cls b/src/cls/IPM/Lifecycle/Module.cls
index 74ec785e..128130c1 100644
--- a/src/cls/IPM/Lifecycle/Module.cls
+++ b/src/cls/IPM/Lifecycle/Module.cls
@@ -98,7 +98,9 @@ Method %PrepareDeploy(ByRef pParams) As %Status
Set tName = $Piece(tResource.Name, ".", 1, *-1)
Set tExtension = $$$ucase($Piece(tResource.Name, ".", *))
- continue:tExtension=""
+ if (tExtension = "") || ($Extract(tExtension) = "/") {
+ continue
+ }
Set tSC = $Case(tExtension,
"GBL":##class(%IPM.Utils.Module).AddGlobalMapping(tPackageDeployNamespace, tName, tSource),
@@ -106,9 +108,9 @@ Method %PrepareDeploy(ByRef pParams) As %Status
"INT":##class(%IPM.Utils.Module).AddRoutineMapping(tPackageDeployNamespace, tName, "INT", tSource),
"MAC":##class(%IPM.Utils.Module).AddRoutineMapping(tPackageDeployNamespace, tName, "MAC", tSource),
"PKG":##class(%IPM.Utils.Module).AddPackageMapping(tPackageDeployNamespace, tName, tSource),
- "":##class(%IPM.Utils.Module).AddRoutineMapping(tPackageDeployNamespace, tName, "ALL", tSource))
+ "":##class(%IPM.Utils.Module).AddRoutineMapping(tPackageDeployNamespace, tName, "ALL", tSource),
+ :$$$OK)
$$$ThrowOnError(tSC)
-
}
Set $Namespace = tPackageDeployNamespace
Write:tVerbose !,"Packaging in namespace: ",$Namespace
From 8ec85da5128b545102c07cb8b99ef02f91abd92d Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:34:51 -0400
Subject: [PATCH 004/182] test: minor sandbox changes
Add git to container
Change to irishealth-community
Install 0.7.1 as a starting point
---
docker-compose.yml | 7 +++++--
tests/sandbox/Dockerfile | 12 +++++++++---
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index f1b5f5b9..10b1e052 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
services:
iris:
build: .
@@ -37,4 +36,8 @@ services:
- IRISNAMESPACE=USER
volumes:
- ./:/home/irisowner/zpm/
- - ./tests/sandbox/:/home/irisowner/sandbox/
\ No newline at end of file
+ - ./tests/sandbox/:/home/irisowner/sandbox/
+ - ./tests/sandbox/fhirpro/:/fhirpro/
+ command:
+ - -a
+ - iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords("*")'
\ No newline at end of file
diff --git a/tests/sandbox/Dockerfile b/tests/sandbox/Dockerfile
index 29c7ee93..378e4321 100644
--- a/tests/sandbox/Dockerfile
+++ b/tests/sandbox/Dockerfile
@@ -1,9 +1,15 @@
-ARG BASE=containers.intersystems.com/intersystems/iris-community:2024.1
+ARG BASE=containers.intersystems.com/intersystems/irishealth-community:2024.1
FROM ${BASE}
USER root
RUN apt-get update \
- && apt-get install -y jq
+ && apt-get install -y jq \
+ && apt-get install -y git
-USER irisowner
\ No newline at end of file
+USER irisowner
+
+RUN --mount=type=bind,src=.,dst=/home/irisowner/sandbox/ \
+ iris start iris && \
+ sh /home/irisowner/sandbox/install-artifact.sh zpm-0.7.1.xml \
+ iris stop iris quietly
\ No newline at end of file
From 71c5cbfb3b7b04d92a5d0dc7af7520e38e0021f5 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 29 Jul 2024 15:45:45 -0400
Subject: [PATCH 005/182] fix: ignore case when checking for file existence
---
src/cls/IPM/ResourceProcessor/Default/Document.cls | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index 783922e9..e4f649d0 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -151,19 +151,19 @@ Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boo
Set tResourcePath = tResourcePath _ tFileExtension
- If ($$$lcase(tFileExtension)'=".xml")&&('##class(%File).Exists(tResourcePath)) {
+ If ($$$lcase(tFileExtension)'=".xml")&&('##class(%IPM.Utils.File).Exists(tResourcePath)) {
Set tResourcePathXML = tResourcePath
Set $PIECE(tResourcePath, ".", *) = "xml"
- If (##class(%File).Exists(tResourcePathXML)) {
+ If (##class(%IPM.Utils.File).Exists(tResourcePathXML)) {
Set tResourcePath = tResourcePathXML
Set ..Format = "XML"
}
}
- If ($$$lcase(tFileExtension)=".mac")&&('##class(%File).Exists(tResourcePath)) {
+ If ($$$lcase(tFileExtension)=".mac")&&('##class(%IPM.Utils.File).Exists(tResourcePath)) {
Set tResourcePathRTN = tResourcePath
Set $Piece(tResourcePathRTN, ".", *) = "rtn"
- If (##class(%File).Exists(tResourcePathRTN)) {
+ If (##class(%IPM.Utils.File).Exists(tResourcePathRTN)) {
Set tResourcePath = tResourcePathRTN
}
}
From 375e46c51ec12b075f3aec4d4e1ae1fbdfd5cc0b Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 30 Jul 2024 13:57:22 -0400
Subject: [PATCH 006/182] enhance: allow specifying pip/pip3 path
---
preload/cls/IPM/Installer.cls | 4 ++++
src/cls/IPM/Lifecycle/Base.cls | 4 ++--
src/cls/IPM/Repo/UniversalSettings.cls | 4 +++-
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index c8aa2751..9b46edba 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -102,6 +102,10 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsAvailable(1, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsTrackingId(pAnalyticsTrackingID, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
+ Set tPipPathKey = ##class(%IPM.Repo.UniversalSettings).#PipPath
+ // Consider detecting whether pip or pip3 is available, and where it is located
+ Set tPipPathValue = $Select($$$isWINDOWS: ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory()), 1: "pip3")
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPipPathKey, tPipPathValue, 0))
Quit $$$OK
}
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 4662e01a..de3534f9 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,8 +621,8 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set command = $ListBuild("pip", "install", "-r", "requirements.txt", "-t", target)
- Set pip = $Select($$$isWINDOWS: ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory()), 1: "pip")
+ Set tPipPathKey = ##class(%IPM.Repo.UniversalSettings).#PipPath
+ Set pip = ##class(%IPM.Repo.UniversalSettings).GetValue(tPipPathKey)
Set command = $ListBuild(pip, "install", "-r", "requirements.txt", "-t", target)
If 'tVerbose {
Set stdout = ""
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index f231b98e..bf5be7d1 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -22,7 +22,9 @@ Parameter TerminalPrompt = "TerminalPrompt";
Parameter PublishTimeout = "publish_timeout";
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout";
+Parameter PipPath = "PipPath";
+
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipPath";
/// Returns configArray, that includes all configurable settings
ClassMethod GetAll(Output configArray) As %Status
From 97e7d3a7d188481221a8268e9290bbb793faffe1 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 31 Jul 2024 14:09:00 -0400
Subject: [PATCH 007/182] docs: add change log for ignore resource casing
---
CHANGELOG.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ad50a60..d9332104 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- HSIEO-10274: Separate DependencyAnalyzer out from IPM
- #261: IPM now truly supports using multiple registries for installation / discovery of packages (without needing to prefix the package with the registry name on "install", although it is still possible and now effective to use the prefix).
- #454: IPM 0.9.x+ uses different globals for storage vs. 0.7.0 and previous. Installation will automatically migrate data from the old globals to the new ones. The old globals are left around in case the user decides to revert to an earlier version.
+- #527: IPM 0.9.x+ ignores the casing of resources when matching files on disk even on case-sensitive filesystems
### Fixed
- HSIEO-9269, HSIEO-9402: % percent perforce directories are no longer necessary
@@ -57,4 +58,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
-
### Deprecated
--
\ No newline at end of file
+-
From 3da42189e5887f9d28d8da711eeb121a62f43494 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 31 Jul 2024 14:11:49 -0400
Subject: [PATCH 008/182] docs: add change log for configuraing pip/pip3 path
---
CHANGELOG.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ad50a60..98f05c7e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
+- #538 Added ability to customize path to `pip3`/`pip` through `config set PipPath` command.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
@@ -57,4 +58,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
-
### Deprecated
--
\ No newline at end of file
+-
From e18f551b6cf511c6fd7c774c77e8b9e82bf1466c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 13:06:50 -0400
Subject: [PATCH 009/182] enhance(pip): use `python -m pip` for installation as
suggested by docs
---
CHANGELOG.md | 2 +-
preload/cls/IPM/Installer.cls | 34 +++++++++++++++++++++++---
src/cls/IPM/Lifecycle/Base.cls | 6 ++---
src/cls/IPM/Repo/UniversalSettings.cls | 4 +--
4 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 98f05c7e..0a2ef563 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
-- #538 Added ability to customize path to `pip3`/`pip` through `config set PipPath` command.
+- #538 Added ability to customize path to `python`/`python3` through `config set PyPath` command.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 9b46edba..651d54d2 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -102,13 +102,39 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsAvailable(1, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsTrackingId(pAnalyticsTrackingID, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
- Set tPipPathKey = ##class(%IPM.Repo.UniversalSettings).#PipPath
- // Consider detecting whether pip or pip3 is available, and where it is located
- Set tPipPathValue = $Select($$$isWINDOWS: ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory()), 1: "pip3")
- $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPipPathKey, tPipPathValue, 0))
+ Set tPyPathKey = ##class(%IPM.Repo.UniversalSettings).#PyPath
+ Set tPyPathValue = ..DetectPython()
+ If tPyPathValue = "" {
+ Write !, "WARNING: Failed to detect path to Python interpreter. You can set it manually using `config set PyPath `"
+ }
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPyPathKey, tPyPathValue, 0))
Quit $$$OK
}
+/// Detect whether python or python3 is available on $PATH
+/// When both are available, use python3
+/// Most of the time, it's equivalent to $Select($$$isWINDOWS:"python", 1:"python3")
+ClassMethod DetectPython() As %String
+{
+ If $$$isWINDOWS {
+ Set tPythons = $lb("python3.exe", "python3", "python.exe", "python")
+ } Else {
+ Set tPythons = $lb("python3", "python")
+ }
+
+ Set ptr = 0
+ While $ListNext(tPythons, ptr, tPy) {
+ Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPython.log""/STDERR=""DetectPython.err"""
+ Set retCode = $ZF(-100, flags, tPy, "--version")
+ If retCode=0 {
+ // TODO resolve absolute path with `python -c "import sys; print(sys.executable)"`
+ Return tPy
+ }
+ }
+
+ Return ""
+}
+
ClassMethod ZPMLoad(pDirectoryName)
{
Quit ##class(%IPM.Main).Shell("load "_pDirectoryName)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index de3534f9..1d9fad28 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,9 +621,9 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set tPipPathKey = ##class(%IPM.Repo.UniversalSettings).#PipPath
- Set pip = ##class(%IPM.Repo.UniversalSettings).GetValue(tPipPathKey)
- Set command = $ListBuild(pip, "install", "-r", "requirements.txt", "-t", target)
+ Set tPyPathKey = ##class(%IPM.Repo.UniversalSettings).#PyPath
+ Set python = ##class(%IPM.Repo.UniversalSettings).GetValue(tPyPathKey)
+ Set command = $ListBuild(python, "-m", "pip", "install", "-r", "requirements.txt", "-t", target)
If 'tVerbose {
Set stdout = ""
}
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index bf5be7d1..118dae2a 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -22,9 +22,9 @@ Parameter TerminalPrompt = "TerminalPrompt";
Parameter PublishTimeout = "publish_timeout";
-Parameter PipPath = "PipPath";
+Parameter PyPath = "PyPath";
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipPath";
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PyPath";
/// Returns configArray, that includes all configurable settings
ClassMethod GetAll(Output configArray) As %Status
From e2183048eaf9825ce67d6ca8b021a26c36126385 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 15:36:08 -0400
Subject: [PATCH 010/182] fix: for windows, try locating `irispip.exe` before
`python.exe -m pip`
---
CHANGELOG.md | 2 +-
preload/cls/IPM/Installer.cls | 49 +++++++++++++++++---------
src/cls/IPM/Lifecycle/Base.cls | 6 ++--
src/cls/IPM/Repo/UniversalSettings.cls | 4 +--
4 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a2ef563..df588e83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
-- #538 Added ability to customize path to `python`/`python3` through `config set PyPath` command.
+- #538 Added ability to customize caller to `pip`. By default will be `/bin/irispip.exe`, `python3 -m pip`, or `python -m pip`, whichever is found first.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 651d54d2..47322823 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -102,39 +102,54 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsAvailable(1, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsTrackingId(pAnalyticsTrackingID, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
- Set tPyPathKey = ##class(%IPM.Repo.UniversalSettings).#PyPath
- Set tPyPathValue = ..DetectPython()
- If tPyPathValue = "" {
- Write !, "WARNING: Failed to detect path to Python interpreter. You can set it manually using `config set PyPath `"
+ Set tPipCallerKey = ##class(%IPM.Repo.UniversalSettings).#PipCaller
+ Set tPipCallerValue = ..DetectPipCaller()
+ If tPipCallerValue = "" {
+ Write !, "WARNING: Failed to detect proper way of calling pip."
+ Write "You can set it manually using `config set PipCaller $lb("""", ""-m"", ""pip"")`"
}
- $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPyPathKey, tPyPathValue, 0))
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPipCallerKey, tPipCallerValue, 0))
Quit $$$OK
}
-/// Detect whether python or python3 is available on $PATH
-/// When both are available, use python3
-/// Most of the time, it's equivalent to $Select($$$isWINDOWS:"python", 1:"python3")
-ClassMethod DetectPython() As %String
+/// Detect the proper call to `pip`. Possible options are `python3 -m pip`, `python -m pip`, `irispip.exe` (for Windows IRIS <= 2024.1).
+/// We prefer `python3 -m pip` over `python -m pip` because on certain legacy systems, `python` might be Python 2.
+ClassMethod DetectPipCaller() As %String
{
+ Write !, "Detecting pip caller"
If $$$isWINDOWS {
- Set tPythons = $lb("python3.exe", "python3", "python.exe", "python")
+ set irispip = ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory())
+ // irispip.exe is the first one we try
+ Set tPipCallers = $lb($lb(irispip), $lb("python3.exe", "-m", "pip"), $lb("python.exe","-m", "pip"))
} Else {
- Set tPythons = $lb("python3", "python")
+ Set tPipCallers = $lb($lb("python3", "-m", "pip"), $lb("python", "-m", "pip"))
}
Set ptr = 0
- While $ListNext(tPythons, ptr, tPy) {
- Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPython.log""/STDERR=""DetectPython.err"""
- Set retCode = $ZF(-100, flags, tPy, "--version")
- If retCode=0 {
- // TODO resolve absolute path with `python -c "import sys; print(sys.executable)"`
- Return tPy
+ Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPipCaller.log""/STDERR=""DetectPipCaller.err"""
+ While $ListNext(tPipCallers, ptr, cmd) {
+ Write !, "Checking """, $LISTTOSTRING(cmd, " "), """"
+ Set program = $List(cmd)
+ Do ..ListToMultiDimensional($List(cmd, 2, *), .args)
+ Set retCode = $ZF(-100, flags, program, .args)
+ If retCode = 0 {
+ Write !, "Success!"
+ Return cmd
}
}
Return ""
}
+ClassMethod ListToMultiDimensional(pList As %List, Output pOutput)
+{
+ Set ptr = 0
+ Kill pOutput
+ While $ListNext(pList, ptr, tItem) {
+ Set pOutput($Increment(pOutput)) = tItem
+ }
+}
+
ClassMethod ZPMLoad(pDirectoryName)
{
Quit ##class(%IPM.Main).Shell("load "_pDirectoryName)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 1d9fad28..0282413e 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,9 +621,9 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set tPyPathKey = ##class(%IPM.Repo.UniversalSettings).#PyPath
- Set python = ##class(%IPM.Repo.UniversalSettings).GetValue(tPyPathKey)
- Set command = $ListBuild(python, "-m", "pip", "install", "-r", "requirements.txt", "-t", target)
+ Set tPipCallerKey = ##class(%IPM.Repo.UniversalSettings).#PipCaller
+ Set tPipCallerValue = ##class(%IPM.Repo.UniversalSettings).GetValue(tPipCallerKey)
+ Set command = tPipCallerValue _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
If 'tVerbose {
Set stdout = ""
}
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index 118dae2a..46e8e5cf 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -22,9 +22,9 @@ Parameter TerminalPrompt = "TerminalPrompt";
Parameter PublishTimeout = "publish_timeout";
-Parameter PyPath = "PyPath";
+Parameter PipCaller = "PipCaller";
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PyPath";
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller";
/// Returns configArray, that includes all configurable settings
ClassMethod GetAll(Output configArray) As %Status
From 8875973c9e2ad5e0e5e95d01999dda709207797d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 15:37:06 -0400
Subject: [PATCH 011/182] feat: add support for printing $listbuild(...) in
universal config
---
src/cls/IPM/Repo/UniversalSettings.cls | 32 ++++++++++++++++++++++++--
1 file changed, 30 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index 46e8e5cf..3b69c864 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -44,7 +44,7 @@ ClassMethod PrintList()
For {
Set k = $Order(confArr(k))
Quit:(k="")
- Write k_": "_confArr(k),!
+ Write k_": "_..PrettyFormat(confArr(k)),!
}
}
@@ -55,7 +55,35 @@ ClassMethod PrintOne(key As %String)
Write "Config key = """_key_""" not found",!
Quit
}
- Write key_":"_..GetValue($Parameter(..%ClassName(1),key)),!
+ Write key_":"_..PrettyFormat(..GetValue($Parameter(..%ClassName(1),key))),!
+}
+
+/// A helper method to format string/number/$lb() recursively
+ClassMethod PrettyFormat(input As %String) As %String
+{
+ If $Data(input) # 2 = 0 { Return "" }
+ If input = $lb() { Return "$lb()" }
+ If input = "" { Return """""" }
+ Try {
+ Set ptr = 0
+ Set output = ""
+ While $ListNext(input, ptr, value) {
+ If output = "" {
+ Set output = "$lb(" _ ..PrettyFormat(value)
+ } Else {
+ Set output = output _ ", " _ ..PrettyFormat(value)
+ }
+ }
+ Set output = output _ ")"
+ Return output
+ } Catch ex {
+ If input = +input {
+ Return input
+ }
+ Return """" _ input _ """"
+ }
+
+ Return "Failed to Format input: " _ input
}
ClassMethod ResetToDefault(key As %String) As %Status
From fb35d0dd14cbeaf0c075255dca69333c85292dea Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 15:50:25 -0400
Subject: [PATCH 012/182] chore(ci): skip 2023.2 due to expired license
---
.github/workflows/images.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
index 97fd5d3d..5774fabb 100644
--- a/.github/workflows/images.yml
+++ b/.github/workflows/images.yml
@@ -30,7 +30,8 @@ jobs:
for tag in $tags
do
# Skip irishealth-community due to bad interaction with ZPM document type
- if test "$n" = "irishealth-community" && test "$tag" = "2023.3"
+ # Also skip 2023.2 because the license has expired
+ if [ "$n" = "irishealth-community" -a test "$tag" = "2023.3" -o "$tag" = "2023.2" ];
then
continue
fi
From d5ff185d84c7ac65b23d7cace95b1ffa598c68de Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 15:53:18 -0400
Subject: [PATCH 013/182] docs: add change log entries for pretty print config
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index df588e83..473c4cca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
- #538 Added ability to customize caller to `pip`. By default will be `/bin/irispip.exe`, `python3 -m pip`, or `python -m pip`, whichever is found first.
+- Added support for `%List` to `config list` and `config get `.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
@@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- HSIEO-10274: Separate DependencyAnalyzer out from IPM
- #261: IPM now truly supports using multiple registries for installation / discovery of packages (without needing to prefix the package with the registry name on "install", although it is still possible and now effective to use the prefix).
- #454: IPM 0.9.x+ uses different globals for storage vs. 0.7.0 and previous. Installation will automatically migrate data from the old globals to the new ones. The old globals are left around in case the user decides to revert to an earlier version.
+- Enclose output of `config list` and `config get ` with quotes.
### Fixed
- HSIEO-9269, HSIEO-9402: % percent perforce directories are no longer necessary
From 2e10191c26e807bfaec1f377b0b7232a897df5b4 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 1 Aug 2024 16:11:37 -0400
Subject: [PATCH 014/182] fix(ci): fix a bug when filtering packages
---
.github/workflows/images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
index 5774fabb..097ce86d 100644
--- a/.github/workflows/images.yml
+++ b/.github/workflows/images.yml
@@ -31,7 +31,7 @@ jobs:
do
# Skip irishealth-community due to bad interaction with ZPM document type
# Also skip 2023.2 because the license has expired
- if [ "$n" = "irishealth-community" -a test "$tag" = "2023.3" -o "$tag" = "2023.2" ];
+ if [ "$n" = "irishealth-community" -a "$tag" = "2023.3" -o "$tag" = "2023.2" ];
then
continue
fi
From 9f70be14f882d268a3a8adcbb3fd27ca0abac974 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 5 Aug 2024 09:21:44 -0400
Subject: [PATCH 015/182] enhance(installer): attempt flexible python on
windows
---
preload/cls/IPM/Installer.cls | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 47322823..58b7efbc 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -118,9 +118,15 @@ ClassMethod DetectPipCaller() As %String
{
Write !, "Detecting pip caller"
If $$$isWINDOWS {
+ If $System.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
+ do ##class(%SYS.Python).GetPythonInfo(.info)
+ If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) {
+ set tInterpreter = ##class(%File).GetDirectory(tPyDylib, 1)_"python.exe"
+ set tFlexiblePython = $lb($lb(tInterpreter, "-m", "pip"))
+ }
+ }
set irispip = ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory())
- // irispip.exe is the first one we try
- Set tPipCallers = $lb($lb(irispip), $lb("python3.exe", "-m", "pip"), $lb("python.exe","-m", "pip"))
+ Set tPipCallers = $lb($lb(irispip))_$Get(tFlexiblePython)_$lb($lb("python3.exe", "-m", "pip"), $lb("python.exe","-m", "pip"))
} Else {
Set tPipCallers = $lb($lb("python3", "-m", "pip"), $lb("python", "-m", "pip"))
}
From 990a0a1e37acebb4a8e70485b58be371dc63b48d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 5 Aug 2024 09:23:53 -0400
Subject: [PATCH 016/182] fix: ensure CPF_PythonRuntimeLibrary is nonempty
---
preload/cls/IPM/Installer.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 58b7efbc..58c9a217 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -120,7 +120,7 @@ ClassMethod DetectPipCaller() As %String
If $$$isWINDOWS {
If $System.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
do ##class(%SYS.Python).GetPythonInfo(.info)
- If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) {
+ If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) && tPyDylib {
set tInterpreter = ##class(%File).GetDirectory(tPyDylib, 1)_"python.exe"
set tFlexiblePython = $lb($lb(tInterpreter, "-m", "pip"))
}
From d57ecfbf9de62b1161c73cc2efc677511fa3741e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 7 Aug 2024 14:27:47 -0400
Subject: [PATCH 017/182] fix: auto detect pip caller unless specifically set
---
CHANGELOG.md | 2 +-
preload/cls/IPM/Installer.cls | 44 +-----------
src/cls/IPM/Lifecycle/Base.cls | 96 +++++++++++++++++++++++++-
src/cls/IPM/Repo/UniversalSettings.cls | 12 +++-
4 files changed, 107 insertions(+), 47 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 473c4cca..cca61c33 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
-- #538 Added ability to customize caller to `pip`. By default will be `/bin/irispip.exe`, `python3 -m pip`, or `python -m pip`, whichever is found first.
+- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
- Added support for `%List` to `config list` and `config get `.
### Changed
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 58c9a217..0ad8320e 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -102,51 +102,11 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsAvailable(1, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetAnalyticsTrackingId(pAnalyticsTrackingID, 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
- Set tPipCallerKey = ##class(%IPM.Repo.UniversalSettings).#PipCaller
- Set tPipCallerValue = ..DetectPipCaller()
- If tPipCallerValue = "" {
- Write !, "WARNING: Failed to detect proper way of calling pip."
- Write "You can set it manually using `config set PipCaller $lb("""", ""-m"", ""pip"")`"
- }
- $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue(tPipCallerKey, tPipCallerValue, 0))
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("PipCaller", "", 0))
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("UseStandalonePip", "", 0))
Quit $$$OK
}
-/// Detect the proper call to `pip`. Possible options are `python3 -m pip`, `python -m pip`, `irispip.exe` (for Windows IRIS <= 2024.1).
-/// We prefer `python3 -m pip` over `python -m pip` because on certain legacy systems, `python` might be Python 2.
-ClassMethod DetectPipCaller() As %String
-{
- Write !, "Detecting pip caller"
- If $$$isWINDOWS {
- If $System.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
- do ##class(%SYS.Python).GetPythonInfo(.info)
- If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) && tPyDylib {
- set tInterpreter = ##class(%File).GetDirectory(tPyDylib, 1)_"python.exe"
- set tFlexiblePython = $lb($lb(tInterpreter, "-m", "pip"))
- }
- }
- set irispip = ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory())
- Set tPipCallers = $lb($lb(irispip))_$Get(tFlexiblePython)_$lb($lb("python3.exe", "-m", "pip"), $lb("python.exe","-m", "pip"))
- } Else {
- Set tPipCallers = $lb($lb("python3", "-m", "pip"), $lb("python", "-m", "pip"))
- }
-
- Set ptr = 0
- Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPipCaller.log""/STDERR=""DetectPipCaller.err"""
- While $ListNext(tPipCallers, ptr, cmd) {
- Write !, "Checking """, $LISTTOSTRING(cmd, " "), """"
- Set program = $List(cmd)
- Do ..ListToMultiDimensional($List(cmd, 2, *), .args)
- Set retCode = $ZF(-100, flags, program, .args)
- If retCode = 0 {
- Write !, "Success!"
- Return cmd
- }
- }
-
- Return ""
-}
-
ClassMethod ListToMultiDimensional(pList As %List, Output pOutput)
{
Set ptr = 0
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 0282413e..dea3dbe7 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,9 +621,7 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set tPipCallerKey = ##class(%IPM.Repo.UniversalSettings).#PipCaller
- Set tPipCallerValue = ##class(%IPM.Repo.UniversalSettings).GetValue(tPipCallerKey)
- Set command = tPipCallerValue _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
+ Set command = ..ResolvePipCaller() _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
If 'tVerbose {
Set stdout = ""
}
@@ -639,6 +637,98 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Quit tSC
}
+Method ResolvePipCaller(ByRef pParams) As %List
+{
+ Set tUseStandalonePip = ##class(%IPM.Repo.UniversalSettings).GetValue("UseStandalonePip")
+ Set tPipCaller = ##class(%IPM.Repo.UniversalSettings).GetValue("PipCaller")
+
+ If tPipCaller '= "" {
+ If tUseStandalonePip = 1 {
+ Return $lb(tPipCaller)
+ } ElseIf tUseStandalonePip = 0 {
+ Return $lb(tPipCaller, "-m", "pip")
+ } Else {
+ Set msg = "Detected PipCaller=""" _ tPipCaller _ """ but UseStandalonePip is not properly set." _ $Char(13, 10)
+ Set msg = msg _ "Please set UseStandalonePip to 1 (if using a pip executable) or 0 (if using a python executable)." _ $Char(13, 10)
+ Set msg = msg _ "Example: zpm ""config set UseStandalonePip 1"""
+ Throw ##class(%Exception.General).%New(msg)
+ }
+ }
+ Return DetectPipCaller(tUseStandalonePip, $Get(pParams("Verbose"), 0))
+}
+
+Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0) As %List
+{
+ Write:pVerbose !,"Detecting pip caller"
+
+ // First try to detect flexible python (in 2024.2 and later)
+ // This is a hack that doesn't always work because the python interpreter may not be in the same directory as the so/dylib/dll.
+ If $System.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
+ Write:pVerbose !, "Attempting to find flexible python... "
+ Do ##class(%SYS.Python).GetPythonInfo(.info)
+ If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) && tPyDylib {
+ // TODO: try `../bin/python3` or `../bin/python` in case the .so is in subfolder `lib`
+ // TODO: try to find `pip3` or `pip` if pUseStandalonePip is 1
+ For filename = "python3", "python" {
+ Set tInterpreter = ##class(%File).GetDirectory(tPyDylib, 1) _ filename
+ If $$$isWINDOWS {
+ Set tInterpreter = tInterpreter_".exe"
+ }
+ If ##class(%File).Exists(tInterpreter) {
+ Write:pVerbose "Success!"
+ Return $lb(tInterpreter, "-m", "pip")
+ }
+ Write:pVerbose "Not Found"
+ }
+ }
+ }
+
+ // For windows, try to find irispip.exe (in 2024.1 and earlier)
+ If $$$isWINDOWS {
+ Write:pVerbose !, "Attempting to find irispip.exe..."
+ Set irispip = ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory())
+ If ##class(%File).Exists(irispip) {
+ Write:pVerbose "Success!"
+ Return $lb(tInterpreter, "-m", "pip")
+ }
+ Write:pVerbose "Not Found"
+ }
+
+ Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPipCaller.log""/STDERR=""DetectPipCaller.err"""
+ // Unless UseStandalonePip is set to 1, try to find python3 or python
+ If pUseStandalonePip '= 1 {
+ Write:pVerbose !, "Attempting to find python3 or python..."
+ For cmd = "python3", "python" {
+ If $$$isWINDOWS { Set cmd = cmd _ ".exe" }
+ Kill args
+ Set args($I(args)) = "-m"
+ Set args($I(args)) = "pip"
+ Set retCode = $ZF(-100, flags, cmd, .args)
+ If retCode = 0 {
+ Write:pVerbose !, "Success!"
+ Return $lb(cmd, "-m", "pip")
+ }
+ Write:pVerbose "Not Found"
+ }
+ }
+
+ // Unless UseStandalonePip is set to 0, try to find pip3 or pip
+ If pUseStandalonePip '= 0 {
+ Write:pVerbose !, "Attempting to find pip3 or pip..."
+ For cmd = "pip3", "pip" {
+ If $$$isWINDOWS { Set cmd = cmd _ ".exe" }
+ Set retCode = $ZF(-100, flags, cmd)
+ If retCode = 0 {
+ Write !, "Success!"
+ Return $lb(cmd)
+ }
+ Write:pVerbose "Not Found"
+ }
+ }
+
+ Throw ##class(%Exception.General).%New("Could not find a suitable pip caller. Consider setting UseStandalonePip and PipCaller")
+}
+
Method %Validate(ByRef pParams) As %Status
{
// NOTE: Resource processor classes and their attributes are validated in OnBeforePhase,
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index 3b69c864..aab205e1 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -22,9 +22,19 @@ Parameter TerminalPrompt = "TerminalPrompt";
Parameter PublishTimeout = "publish_timeout";
+/// A path pointing either to a python executable or a pip executable, controlled by #UseStandalonePip
+/// In the special case where the path is empty, the script tries to resolve the path in the following order:
+/// 1. Where available (typically 2024.2+) - Use PythonRuntimeLibrary in iris.cpf to find the directory containing the python executable
+/// 2. On Windows, try to find /bin/irispip.exe
+/// 3. Unless UseStandalonePip is explicitly set to 1, try to find whichever python3/python is first available in $PATH
+/// 4. Unless UseStandalonePip is explicitly set to 0, try to find whichever pip3/pip is first available in $PATH
Parameter PipCaller = "PipCaller";
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller";
+/// Possible values: "", 0, 1
+/// Indicates whether PipCaller is a pip executable instead of python
+Parameter UseStandalonePip = "UseStandalonePip";
+
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip";
/// Returns configArray, that includes all configurable settings
ClassMethod GetAll(Output configArray) As %Status
From 4f48a191fa0a15b1245473e7142a56393bacd7bd Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 7 Aug 2024 14:28:58 -0400
Subject: [PATCH 018/182] Revert "feat: add support for printing
$listbuild(...) in universal config"
This reverts commit 8875973c9e2ad5e0e5e95d01999dda709207797d.
---
src/cls/IPM/Repo/UniversalSettings.cls | 32 ++------------------------
1 file changed, 2 insertions(+), 30 deletions(-)
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index aab205e1..da0308e0 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -54,7 +54,7 @@ ClassMethod PrintList()
For {
Set k = $Order(confArr(k))
Quit:(k="")
- Write k_": "_..PrettyFormat(confArr(k)),!
+ Write k_": "_confArr(k),!
}
}
@@ -65,35 +65,7 @@ ClassMethod PrintOne(key As %String)
Write "Config key = """_key_""" not found",!
Quit
}
- Write key_":"_..PrettyFormat(..GetValue($Parameter(..%ClassName(1),key))),!
-}
-
-/// A helper method to format string/number/$lb() recursively
-ClassMethod PrettyFormat(input As %String) As %String
-{
- If $Data(input) # 2 = 0 { Return "" }
- If input = $lb() { Return "$lb()" }
- If input = "" { Return """""" }
- Try {
- Set ptr = 0
- Set output = ""
- While $ListNext(input, ptr, value) {
- If output = "" {
- Set output = "$lb(" _ ..PrettyFormat(value)
- } Else {
- Set output = output _ ", " _ ..PrettyFormat(value)
- }
- }
- Set output = output _ ")"
- Return output
- } Catch ex {
- If input = +input {
- Return input
- }
- Return """" _ input _ """"
- }
-
- Return "Failed to Format input: " _ input
+ Write key_":"_..GetValue($Parameter(..%ClassName(1),key)),!
}
ClassMethod ResetToDefault(key As %String) As %Status
From 48687147e2340245679abce8225b9cce574877bf Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 7 Aug 2024 14:29:30 -0400
Subject: [PATCH 019/182] Revert "docs: add change log entries for pretty print
config"
This reverts commit d5ff185d84c7ac65b23d7cace95b1ffa598c68de.
---
CHANGELOG.md | 2 --
1 file changed, 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cca61c33..bc202cae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
-- Added support for `%List` to `config list` and `config get `.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
@@ -20,7 +19,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- HSIEO-10274: Separate DependencyAnalyzer out from IPM
- #261: IPM now truly supports using multiple registries for installation / discovery of packages (without needing to prefix the package with the registry name on "install", although it is still possible and now effective to use the prefix).
- #454: IPM 0.9.x+ uses different globals for storage vs. 0.7.0 and previous. Installation will automatically migrate data from the old globals to the new ones. The old globals are left around in case the user decides to revert to an earlier version.
-- Enclose output of `config list` and `config get ` with quotes.
### Fixed
- HSIEO-9269, HSIEO-9402: % percent perforce directories are no longer necessary
From 3ccd7800a447e8466baf964b5a8b726178a992ed Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 7 Aug 2024 14:31:37 -0400
Subject: [PATCH 020/182] style: remove unused classmethod
ListToMultiDimensional
---
preload/cls/IPM/Installer.cls | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 0ad8320e..c7935511 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -107,15 +107,6 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
Quit $$$OK
}
-ClassMethod ListToMultiDimensional(pList As %List, Output pOutput)
-{
- Set ptr = 0
- Kill pOutput
- While $ListNext(pList, ptr, tItem) {
- Set pOutput($Increment(pOutput)) = tItem
- }
-}
-
ClassMethod ZPMLoad(pDirectoryName)
{
Quit ##class(%IPM.Main).Shell("load "_pDirectoryName)
From 8b8631ae3a363909314ce43ba8dd3b4710f6bb57 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 8 Aug 2024 15:36:37 -0400
Subject: [PATCH 021/182] fix: fix call to DetectPipCaller
---
src/cls/IPM/Lifecycle/Base.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index dea3dbe7..ff9b1336 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -654,7 +654,7 @@ Method ResolvePipCaller(ByRef pParams) As %List
Throw ##class(%Exception.General).%New(msg)
}
}
- Return DetectPipCaller(tUseStandalonePip, $Get(pParams("Verbose"), 0))
+ Return ..DetectPipCaller(tUseStandalonePip, $Get(pParams("Verbose"), 0))
}
Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0) As %List
From 36ddc87396f9849e8858bcc623a99ba1eae74dd1 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 8 Aug 2024 15:48:20 -0400
Subject: [PATCH 022/182] fix: pass pParams correctly to ResolvePipCaller
---
src/cls/IPM/Lifecycle/Base.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index ff9b1336..b24514cc 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,7 +621,7 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set command = ..ResolvePipCaller() _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
+ Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
If 'tVerbose {
Set stdout = ""
}
From df00e1650a79bcb2ed150386c240b9eb1a6aebcf Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 13 Aug 2024 10:45:25 -0400
Subject: [PATCH 023/182] style: remove postconditionals and use 1 command per
line
---
src/cls/IPM/Lifecycle/Base.cls | 60 +++++++++++++++++++++++++---------
1 file changed, 44 insertions(+), 16 deletions(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index b24514cc..f42e5a4d 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -659,12 +659,16 @@ Method ResolvePipCaller(ByRef pParams) As %List
Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0) As %List
{
- Write:pVerbose !,"Detecting pip caller"
+ If pVerbose {
+ Write !,"Detecting pip caller"
+ }
// First try to detect flexible python (in 2024.2 and later)
// This is a hack that doesn't always work because the python interpreter may not be in the same directory as the so/dylib/dll.
If $System.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
- Write:pVerbose !, "Attempting to find flexible python... "
+ If pVerbose {
+ Write !, "Attempting to find flexible python... "
+ }
Do ##class(%SYS.Python).GetPythonInfo(.info)
If $Data(info("CPF_PythonRuntimeLibrary"), tPyDylib) && tPyDylib {
// TODO: try `../bin/python3` or `../bin/python` in case the .so is in subfolder `lib`
@@ -675,54 +679,78 @@ Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0)
Set tInterpreter = tInterpreter_".exe"
}
If ##class(%File).Exists(tInterpreter) {
- Write:pVerbose "Success!"
+ If pVerbose{
+ Write "Success!"
+ }
Return $lb(tInterpreter, "-m", "pip")
}
- Write:pVerbose "Not Found"
+ If pVerbose {
+ Write "Not Found"
+ }
}
}
- }
+ }
// For windows, try to find irispip.exe (in 2024.1 and earlier)
If $$$isWINDOWS {
- Write:pVerbose !, "Attempting to find irispip.exe..."
+ If pVerbose {
+ Write !, "Attempting to find irispip.exe..."
+ }
Set irispip = ##class(%File).NormalizeFilename("irispip.exe", $System.Util.BinaryDirectory())
If ##class(%File).Exists(irispip) {
- Write:pVerbose "Success!"
+ If pVerbose {
+ Write "Success!"
+ }
Return $lb(tInterpreter, "-m", "pip")
}
- Write:pVerbose "Not Found"
+ If pVerbose {
+ Write "Not Found"
+ }
}
Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPipCaller.log""/STDERR=""DetectPipCaller.err"""
// Unless UseStandalonePip is set to 1, try to find python3 or python
If pUseStandalonePip '= 1 {
- Write:pVerbose !, "Attempting to find python3 or python..."
+ If pVerbose {
+ Write !, "Attempting to find python3 or python..."
+ }
For cmd = "python3", "python" {
- If $$$isWINDOWS { Set cmd = cmd _ ".exe" }
+ If $$$isWINDOWS {
+ Set cmd = cmd _ ".exe"
+ }
Kill args
Set args($I(args)) = "-m"
Set args($I(args)) = "pip"
Set retCode = $ZF(-100, flags, cmd, .args)
If retCode = 0 {
- Write:pVerbose !, "Success!"
+ If pVerbose {
+ Write "Success!"
+ }
Return $lb(cmd, "-m", "pip")
}
- Write:pVerbose "Not Found"
+ If pVerbose {
+ Write "Not Found"
+ }
}
}
// Unless UseStandalonePip is set to 0, try to find pip3 or pip
If pUseStandalonePip '= 0 {
- Write:pVerbose !, "Attempting to find pip3 or pip..."
+ If pVerbose {
+ Write !, "Attempting to find pip3 or pip..."
+ }
For cmd = "pip3", "pip" {
- If $$$isWINDOWS { Set cmd = cmd _ ".exe" }
+ If $$$isWINDOWS {
+ Set cmd = cmd _ ".exe"
+ }
Set retCode = $ZF(-100, flags, cmd)
If retCode = 0 {
- Write !, "Success!"
+ Write "Success!"
Return $lb(cmd)
}
- Write:pVerbose "Not Found"
+ If pVerbose {
+ Write "Not Found"
+ }
}
}
From 7c5cc6abd38293559da602aa671ffc085e95caf3 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 13 Aug 2024 10:47:50 -0400
Subject: [PATCH 024/182] docs: add parameter link in class referece
---
src/cls/IPM/Repo/UniversalSettings.cls | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index da0308e0..b360a19e 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -22,16 +22,16 @@ Parameter TerminalPrompt = "TerminalPrompt";
Parameter PublishTimeout = "publish_timeout";
-/// A path pointing either to a python executable or a pip executable, controlled by #UseStandalonePip
+/// A path pointing either to a python executable or a pip executable, controlled by UseStandalonePip
/// In the special case where the path is empty, the script tries to resolve the path in the following order:
/// 1. Where available (typically 2024.2+) - Use PythonRuntimeLibrary in iris.cpf to find the directory containing the python executable
/// 2. On Windows, try to find /bin/irispip.exe
-/// 3. Unless UseStandalonePip is explicitly set to 1, try to find whichever python3/python is first available in $PATH
-/// 4. Unless UseStandalonePip is explicitly set to 0, try to find whichever pip3/pip is first available in $PATH
+/// 3. Unless UseStandalonePip is explicitly set to 1, try to find whichever python3/python is first available in $PATH
+/// 4. Unless UseStandalonePip is explicitly set to 0, try to find whichever pip3/pip is first available in $PATH
Parameter PipCaller = "PipCaller";
/// Possible values: "", 0, 1
-/// Indicates whether PipCaller is a pip executable instead of python
+/// Indicates whether PipCaller is a pip executable instead of python
Parameter UseStandalonePip = "UseStandalonePip";
Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip";
From e74d83528ce15dc59c3ca98ef7f87bb65530bb5d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 19 Aug 2024 16:14:46 -0400
Subject: [PATCH 025/182] fix: specify python version for wheel ABI
compatibility
---
src/cls/IPM/Lifecycle/Base.cls | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index f42e5a4d..1cf10823 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -621,7 +621,15 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Write:tVerbose !
Set target = ##class(%File).NormalizeDirectory("python", $System.Util.ManagerDirectory())
- Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target)
+ If '$System.CLS.IsMthd("%SYS.Python", "Import") {
+ Throw ##class(%Exception.General).%New("Embedded Python is not available in this instance.")
+ }
+ Set tSysModule = ##class(%SYS.Python).Import("sys")
+ Set tPyMajor = tSysModule."version_info".major
+ Set tPyMinor = tSysModule."version_info".minor
+ Set tPyMicro = tSysModule."version_info".micro
+ Set tPyVersion = tPyMajor_"."_tPyMinor_"."_tPyMicro
+ Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target, "--python-version", tPyVersion, "--only-binary=:all:")
If 'tVerbose {
Set stdout = ""
}
From bb6f02935dffafad7739217a8d21a975927916e5 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:14:26 -0400
Subject: [PATCH 026/182] fix: actually use irispip
---
src/cls/IPM/Lifecycle/Base.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 1cf10823..c6d64f94 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -709,7 +709,7 @@ Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0)
If pVerbose {
Write "Success!"
}
- Return $lb(tInterpreter, "-m", "pip")
+ Return $lb(irispip)
}
If pVerbose {
Write "Not Found"
From 55fc3378c3d63100b15a0443255feb229accce29 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 20 Aug 2024 14:54:39 -0400
Subject: [PATCH 027/182] enhance: use existent `RunCommand` and better verbose
settings
---
src/cls/IPM/Lifecycle/Base.cls | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index c6d64f94..a71247a2 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -630,7 +630,11 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Set tPyMicro = tSysModule."version_info".micro
Set tPyVersion = tPyMajor_"."_tPyMinor_"."_tPyMicro
Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target, "--python-version", tPyVersion, "--only-binary=:all:")
- If 'tVerbose {
+ If tVerbose {
+ Write !, "Running "
+ Zwrite command
+ }
+ Else{
Set stdout = ""
}
Set tSC = ##class(%IPM.Utils.Module).RunCommand(pRoot, command,.stdout)
@@ -716,7 +720,6 @@ Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0)
}
}
- Set flags = "/SHELL/LOGCMD/STDOUT=""DetectPipCaller.log""/STDERR=""DetectPipCaller.err"""
// Unless UseStandalonePip is set to 1, try to find python3 or python
If pUseStandalonePip '= 1 {
If pVerbose {
@@ -726,15 +729,14 @@ Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0)
If $$$isWINDOWS {
Set cmd = cmd _ ".exe"
}
- Kill args
- Set args($I(args)) = "-m"
- Set args($I(args)) = "pip"
- Set retCode = $ZF(-100, flags, cmd, .args)
- If retCode = 0 {
+ set cmd = $lb(cmd, "-m", "pip")
+ Set cwd = ##class(%SYSTEM.Util).InstallDirectory()
+ Set tSC = ##class(%IPM.Utils.Module).RunCommand(cwd, cmd, "")
+ If $$$ISOK(tSC) {
If pVerbose {
Write "Success!"
}
- Return $lb(cmd, "-m", "pip")
+ Return cmd
}
If pVerbose {
Write "Not Found"
@@ -751,10 +753,14 @@ Method DetectPipCaller(pUseStandalonePip As %Boolean, pVerbose As %Boolean = 0)
If $$$isWINDOWS {
Set cmd = cmd _ ".exe"
}
- Set retCode = $ZF(-100, flags, cmd)
- If retCode = 0 {
- Write "Success!"
- Return $lb(cmd)
+ Set cmd = $lb(cmd)
+ Set cwd = ##class(%SYSTEM.Util).InstallDirectory()
+ Set tSC = ##class(%IPM.Utils.Module).RunCommand(cwd, cmd, "")
+ If $$$ISOK(tSC) {
+ If pVerbose {
+ Write "Success!"
+ }
+ Return cmd
}
If pVerbose {
Write "Not Found"
From c099b3a004c00efcb44be6da9288ebb187ac2f85 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 26 Aug 2024 10:50:41 -0400
Subject: [PATCH 028/182] ci: trigger workflow on completion of packages test
---
.github/workflows/packages-summary.yml | 18 +++++-------------
1 file changed, 5 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/packages-summary.yml b/.github/workflows/packages-summary.yml
index a25f1307..0b647b80 100644
--- a/.github/workflows/packages-summary.yml
+++ b/.github/workflows/packages-summary.yml
@@ -1,17 +1,9 @@
name: Summarize package test results
on:
- workflow_call:
- inputs:
- runId:
- type: number
- required: true
- description: "Id of the workflow run to summarize"
- workflow_dispatch:
- inputs:
- runId:
- type: number
- required: true
- description: "Id of the workflow run to summarize"
+ workflow_run:
+ workflows: ["Test major packages"]
+ types:
+ - completed
jobs:
summarize:
@@ -21,7 +13,7 @@ jobs:
steps:
- name: Donwload log zip
run: |
- curl -L -o logs.zip "${{ env.BASE_URL }}/${{ inputs.runId }}/logs" -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}'
+ curl -L -o logs.zip "${{ env.BASE_URL }}/${{ github.event.workflow_run.id }}/logs" -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}'
- name: Unzip logs
run: |
unzip logs.zip -d /tmp/logs/
From ff6962d535f08801cfcb20dcc171a2f706dbfd40 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 26 Aug 2024 11:30:09 -0400
Subject: [PATCH 029/182] ci(packages): remove call to self-summarization
---
.github/workflows/packages.yml | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 0f54b0d0..253e8aba 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -134,14 +134,4 @@ jobs:
echo "::endgroup::"
done
- IFS=' '
-
- summarize:
- needs:
- - run-tests
- runs-on: ubuntu-latest
- steps:
- - name: Call package-summary workflow
- uses: ./.github/workflows/packages-summary.yml
- with:
- runId: ${{ github.run_id }}
+ IFS=' '
\ No newline at end of file
From ad022cddbc209672e575670e33bf9edc5d4279f0 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 27 Aug 2024 09:16:37 -0400
Subject: [PATCH 030/182] fix: remove debug code, add IPM ver arg
---
docker-compose.yml | 1 -
tests/sandbox/Dockerfile | 5 +++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 10b1e052..dea11a7a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -37,7 +37,6 @@ services:
volumes:
- ./:/home/irisowner/zpm/
- ./tests/sandbox/:/home/irisowner/sandbox/
- - ./tests/sandbox/fhirpro/:/fhirpro/
command:
- -a
- iris session iris -U%SYS '##class(Security.Users).UnExpireUserPasswords("*")'
\ No newline at end of file
diff --git a/tests/sandbox/Dockerfile b/tests/sandbox/Dockerfile
index 378e4321..8e930e40 100644
--- a/tests/sandbox/Dockerfile
+++ b/tests/sandbox/Dockerfile
@@ -1,5 +1,6 @@
-ARG BASE=containers.intersystems.com/intersystems/irishealth-community:2024.1
+ARG BASE=containers.intersystems.com/intersystems/iris-community:2024.1
FROM ${BASE}
+ARG IPM_VERSION=zpm-0.7.2.xml
USER root
@@ -11,5 +12,5 @@ USER irisowner
RUN --mount=type=bind,src=.,dst=/home/irisowner/sandbox/ \
iris start iris && \
- sh /home/irisowner/sandbox/install-artifact.sh zpm-0.7.1.xml \
+ sh /home/irisowner/sandbox/install-artifact.sh $IPM_VERSION \
iris stop iris quietly
\ No newline at end of file
From 34cb44d2f7d561db69e4022a2586bba12f5106c1 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Fri, 5 Jul 2024 13:06:31 -0400
Subject: [PATCH 031/182] fix: #{expression} in module.xml works again
---
src/cls/IPM/Storage/Module.cls | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls
index f3cf71b3..d7c14613 100644
--- a/src/cls/IPM/Storage/Module.cls
+++ b/src/cls/IPM/Storage/Module.cls
@@ -1434,9 +1434,26 @@ Method %Evaluate(pAttrValue As %String, ByRef pParams) As %String [ Internal ]
Set name = ..Name
Set attrValue = ##class(%Regex.Matcher).%New("(?i)\{name\}", attrValue).ReplaceAll($Replace(name,"\","\\"))
+ Set regex = ##class(%Regex.Matcher).%New("#\{([^}]+)\}", tAttrValue)
+ While regex.Locate() {
+ Set expr = regex.Group(1)
+ Set value = ..%EvaluateExpression(expr)
+ Set $Extract(attrValue, regex.Start, regex.End - 1) = value
+ Set regex.Text = attrValue
+ }
+
Return attrValue
}
+Method %EvaluateExpression(pExpr) As %String [ Internal ]
+{
+ Try {
+ return @pExpr
+ } Catch ex {
+ }
+ Return ""
+}
+
Storage Default
{
@@ -1541,3 +1558,4 @@ Storage Default
}
}
+
From cd7f19b487d9b8271e1745888f00c72a771e5d71 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 27 Aug 2024 11:26:43 -0400
Subject: [PATCH 032/182] test: add unit test for expression
---
tests/unit_tests/Test/PM/Unit/Module.cls | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/unit_tests/Test/PM/Unit/Module.cls b/tests/unit_tests/Test/PM/Unit/Module.cls
index 10f7a482..b3269d9d 100644
--- a/tests/unit_tests/Test/PM/Unit/Module.cls
+++ b/tests/unit_tests/Test/PM/Unit/Module.cls
@@ -4,6 +4,7 @@ Class Test.PM.Unit.Module Extends %UnitTest.TestCase
Method TestEvaluateAttribute()
{
Set tModule = ##class(%IPM.Storage.Module).%New()
+ Set tModule.Root = "/tmp/foo/bar"
Set tests = $ListBuild(
"${cspdir}","{$cspdir}",
@@ -23,6 +24,7 @@ Method TestEvaluateAttribute()
Set test = $ListGet(tests, i)
Do $$$AssertNotTrue(tModule.%Evaluate(test)=test, test)
}
+ Do $$$AssertEquals(tModule.%Evaluate("#{..Root}"),tModule.Root)
}
}
From 7cb4efafe55f1a65e1cb412764f25e51f51b0af0 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 27 Aug 2024 14:47:19 -0400
Subject: [PATCH 033/182] fix: consistency in treatment of top-level Mapping
---
src/cls/IPM/StudioDocument/Module.cls | 9 +++-
.../Test/PM/Integration/Mappings.cls | 41 +++++++++++++++++++
.../cls/MappingTestUnwrapped/Sample.cls | 4 ++
.../_data/mapping-test-unwrapped/module.xml | 11 +++++
.../cls/MappingTestWrapped/Sample.cls | 4 ++
.../_data/mapping-test-wrapped/module.xml | 15 +++++++
6 files changed, 82 insertions(+), 2 deletions(-)
create mode 100644 tests/integration_tests/Test/PM/Integration/Mappings.cls
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/cls/MappingTestUnwrapped/Sample.cls
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/module.xml
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/cls/MappingTestWrapped/Sample.cls
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/module.xml
diff --git a/src/cls/IPM/StudioDocument/Module.cls b/src/cls/IPM/StudioDocument/Module.cls
index ce7137e7..e5659da4 100644
--- a/src/cls/IPM/StudioDocument/Module.cls
+++ b/src/cls/IPM/StudioDocument/Module.cls
@@ -496,6 +496,11 @@ XData InternalXSL
+
+
+
+
+
@@ -507,8 +512,8 @@ XData InternalXSL
-
-
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/Mappings.cls b/tests/integration_tests/Test/PM/Integration/Mappings.cls
new file mode 100644
index 00000000..e52a3f5d
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/Mappings.cls
@@ -0,0 +1,41 @@
+Class Test.PM.Integration.Mappings Extends %UnitTest.TestCase
+{
+
+Method TestWrapped()
+{
+ Do ..RunTestCase("_data/mapping-test-wrapped","MappingTestWrapped")
+}
+
+Method TestUnwrappedWrapped()
+{
+ Do ..RunTestCase("_data/mapping-test-unwrapped","MappingTestUnwrapped")
+}
+
+Method RunTestCase(subdirectory As %String, packageName As %String)
+{
+ Try {
+ Set tTestRoot = $Get(^UnitTestRoot)
+ Set tParams("Verbose") = 1
+ Set tModuleDirectory = ##class(%File).NormalizeDirectory(subdirectory, tTestRoot)
+ $$$ThrowOnError(##class(%IPM.Utils.Module).LoadModuleFromDirectory(tModuleDirectory,.tParams))
+ Set module = ##class(%IPM.Storage.Module).NameOpen(packageName,,.sc)
+ $$$ThrowOnError(sc)
+ Do $$$AssertEquals(module.Mappings.Count(),1)
+
+ Do ..AssertPackageMappingExists(packageName_".Foo")
+ } Catch e {
+ Do $$$AssertFailure("An exception occurred: "_$System.Status.GetErrorText(e.AsStatus()))
+ }
+}
+
+Method AssertPackageMappingExists(mappingName As %String)
+{
+ Set namespace = $namespace
+ New $namespace
+ Set $namespace = "%SYS"
+ Set mappingCreated = ##class(Config.MapPackages).Exists("USER-VERIFY","MappingTestUnwrapped.Foo")
+ Set $namespace = namespace
+ Do $$$AssertTrue(mappingCreated)
+}
+
+}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/cls/MappingTestUnwrapped/Sample.cls b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/cls/MappingTestUnwrapped/Sample.cls
new file mode 100644
index 00000000..4faf1ecd
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/cls/MappingTestUnwrapped/Sample.cls
@@ -0,0 +1,4 @@
+Class MappingTestUnwrapped.Sample Extends %RegisteredObject
+{
+
+}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/module.xml b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/module.xml
new file mode 100644
index 00000000..ff541282
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-unwrapped/module.xml
@@ -0,0 +1,11 @@
+
+
+
+ MappingTestUnwrapped
+ 0.0.1+snapshot
+ module
+
+
+ Module
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/cls/MappingTestWrapped/Sample.cls b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/cls/MappingTestWrapped/Sample.cls
new file mode 100644
index 00000000..2e7fd82c
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/cls/MappingTestWrapped/Sample.cls
@@ -0,0 +1,4 @@
+Class MappingTestWrapped.Sample Extends %RegisteredObject
+{
+
+}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/module.xml b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/module.xml
new file mode 100644
index 00000000..780582fb
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/mapping-test-wrapped/module.xml
@@ -0,0 +1,15 @@
+
+
+
+ MappingTestWrapped
+ 0.0.1+snapshot
+ module
+
+
+
+
+
+
+ Module
+
+
From 6b93ee6f4ea9ea0827606a32fff796eb228c550a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 28 Aug 2024 15:20:23 -0400
Subject: [PATCH 034/182] fix: use semver 2.0 to exclude snapshots in range max
---
.../General/SemanticVersionExpression/Range.cls | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/General/SemanticVersionExpression/Range.cls b/src/cls/IPM/General/SemanticVersionExpression/Range.cls
index bda2e4f1..81b0cdd5 100644
--- a/src/cls/IPM/General/SemanticVersionExpression/Range.cls
+++ b/src/cls/IPM/General/SemanticVersionExpression/Range.cls
@@ -118,7 +118,7 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
Set tMajor = $Piece(tExpr,".",1)
Set tMinor = $Piece(tExpr,".",2)
- Set tComparators = tComparators_$ListBuild("<"_tMajor_"."_(tMinor+1)_".0")
+ Set tComparators = tComparators_$ListBuild("<"_tMajor_"."_(tMinor+1)_".0-0")
}
} ElseIf tIsCaretRange {
// Caret ranges:
@@ -134,7 +134,7 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
Set tPatch = $Piece(tMajorMinorPatch,".",3)
If (tDotLength < 3) {
If (tMajor '= 0) && 'tIsXRange {
- Set tComparators = tComparators_$ListBuild(">="_$Replace(tExpr,".x",".0"),"<"_(tMajor+1)_".0.0")
+ Set tComparators = tComparators_$ListBuild(">="_$Replace(tExpr,".x",".0"),"<"_(tMajor+1)_".0.0-0")
Set tIsXRange = 0
} Else {
// Detected and properly handled by X-range.
@@ -146,12 +146,12 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
Set tMax = ""
If (+tMajor = 0) && (tMinor '= "x") {
If (+tMinor = 0) && (tPatch '= "x") {
- Set tMax = "0.0."_(tPatch+1)
+ Set tMax = "0.0."_(tPatch+1)_"-0"
} Else {
- Set tMax = "0."_(tMinor+1)_".0"
+ Set tMax = "0."_(tMinor+1)_".0-0"
}
} Else {
- Set tMax = (tMajor+1)_".0.0"
+ Set tMax = (tMajor+1)_".0.0-0"
}
// Maximum
@@ -174,9 +174,9 @@ ClassMethod FromString(pRangeExpr As %String, Output pRange As %IPM.General.Sema
// Accept anything!
Set tComparators = tComparators_$ListBuild(">=0.0.0")
} ElseIf (tMinor = "") || (tMinor = "x") {
- Set tComparators = tComparators_$ListBuild(">="_tMajor_".0.0","<"_(tMajor+1)_".0.0")
+ Set tComparators = tComparators_$ListBuild(">="_tMajor_".0.0","<"_(tMajor+1)_".0.0-0")
} ElseIf (tPatch = "") || (tPatch = "x") {
- Set tComparators = tComparators_$ListBuild(">="_tMajor_"."_tMinor_".0","<"_tMajor_"."_(tMinor+1)_".0")
+ Set tComparators = tComparators_$ListBuild(">="_tMajor_"."_tMinor_".0","<"_tMajor_"."_(tMinor+1)_".0-0")
}
}
}
From 360152eae49cdbe9e47898c4553b1de9b79b5900 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 28 Aug 2024 15:20:47 -0400
Subject: [PATCH 035/182] test: adjust testcases for semver 2.0
---
.../Test/PM/Unit/SemVer/Expressions.cls | 62 +++++++++++--------
1 file changed, 37 insertions(+), 25 deletions(-)
diff --git a/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls b/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
index 49a16bb1..dddddefc 100644
--- a/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
+++ b/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
@@ -34,13 +34,25 @@ Method TestPrerelease()
Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.1")
Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.2")
Do ..AssertSatisfied(">=1.0.0-alpha.1","1.0.0-beta.1")
- Do ..AssertSatisfied("^1.0.0","1.0.0-1.m1")
+ Do ..AssertNotSatisfied("^1.0.0","1.0.0-1.m1") // This shouldn't be satisfied because 1.0.0-1.m1 is a prerelease
Do ..AssertNotSatisfied(">=1.0.0-beta.2","1.0.0-beta.1")
Do ..AssertNotSatisfied(">=1.0.0-beta.1","1.0.0-alpha.1")
// This is the problematic case without HSIEO-10520
Do ..AssertNotSatisfied("1.0.0-1.m1","1.0.0")
Do ..AssertNotSatisfied("=1.0.0-1.m1","1.0.0")
+
+ // Issue #557: snapshots/prereleases shouldn't be considered as a lower version number
+ // Even though 1.5.0-beta.1 is technically within the 1.x range, it's a prerelease and should not be considered satisfiable.
+ // Similarly 1.1.5-beta.1 does not satisify 1.1.x. See https://github.com/npm/node-semver?tab=readme-ov-file#prerelease-tags
+ Do ..AssertNotSatisfied("1.x", "1.5.0-beta.1")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-0")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-SNAPSHOT")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-alpha")
+ Do ..AssertNotSatisfied("1.1.x", "1.1.5-beta.1")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-0")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-SNAPSHOT")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-alpha")
}
Method TestEquivalenceHyphenRanges()
@@ -53,38 +65,38 @@ Method TestEquivalenceHyphenRanges()
Method TestEquivalenceXRanges()
{
- Do ..AssertEquivalent("*", ">=0.0.0")
- Do ..AssertEquivalent("", ">=0.0.0")
- Do ..AssertEquivalent("1", ">=1.0.0 <2.0.0")
- Do ..AssertEquivalent("1.x", ">=1.0.0 <2.0.0")
- Do ..AssertEquivalent("1.2", ">=1.2.0 <1.3.0")
- Do ..AssertEquivalent("1.2.*", ">=1.2.0 <1.3.0")
- Do ..AssertEquivalent("1.2.X", ">=1.2.0 <1.3.0")
+ Do ..AssertEquivalent("*", ">=0.0.0") // Do we want to allow pre-release versions?
+ Do ..AssertEquivalent("", ">=0.0.0") // Do we want to allow pre-release versions?
+ Do ..AssertEquivalent("1", ">=1.0.0 <2.0.0-0")
+ Do ..AssertEquivalent("1.x", ">=1.0.0 <2.0.0-0")
+ Do ..AssertEquivalent("1.2", ">=1.2.0 <1.3.0-0")
+ Do ..AssertEquivalent("1.2.*", ">=1.2.0 <1.3.0-0")
+ Do ..AssertEquivalent("1.2.X", ">=1.2.0 <1.3.0-0")
}
Method TestEquivalenceTildeRanges()
{
- Do ..AssertEquivalent("~1.2.3", ">=1.2.3 <1.3.0")
- Do ..AssertEquivalent("~1.2", ">=1.2.0 <1.3.0")
- Do ..AssertEquivalent("~1", ">=1.0.0 <2.0.0")
- Do ..AssertEquivalent("~0.2.3", ">=0.2.3 <0.3.0")
- Do ..AssertEquivalent("~0.2", ">=0.2.0 <0.3.0")
- Do ..AssertEquivalent("~0", ">=0.0.0 <1.0.0")
- Do ..AssertEquivalent("~1.2.3-beta.2", ">=1.2.3-beta.2 <1.3.0")
+ Do ..AssertEquivalent("~1.2.3", ">=1.2.3 <1.3.0-0")
+ Do ..AssertEquivalent("~1.2", ">=1.2.0 <1.3.0-0")
+ Do ..AssertEquivalent("~1", ">=1.0.0 <2.0.0-0")
+ Do ..AssertEquivalent("~0.2.3", ">=0.2.3 <0.3.0-0")
+ Do ..AssertEquivalent("~0.2", ">=0.2.0 <0.3.0-0")
+ Do ..AssertEquivalent("~0", ">=0.0.0 <1.0.0-0")
+ Do ..AssertEquivalent("~1.2.3-beta.2", ">=1.2.3-beta.2 <1.3.0-0")
}
Method TestEquivalenceCaretRanges()
{
- Do ..AssertEquivalent("^1.2.3", ">=1.2.3 <2.0.0")
- Do ..AssertEquivalent("^0.2.3", ">=0.2.3 <0.3.0")
- Do ..AssertEquivalent("^0.0.3", ">=0.0.3 <0.0.4")
- Do ..AssertEquivalent("^1.2.3-beta.2", ">=1.2.3-beta.2 <2.0.0")
- Do ..AssertEquivalent("^0.0.3-beta", ">=0.0.3-beta <0.0.4")
- Do ..AssertEquivalent("^1.2.x", ">=1.2.0 <2.0.0")
- Do ..AssertEquivalent("^0.0.x", ">=0.0.0 <0.1.0")
- Do ..AssertEquivalent("^0.0", ">=0.0.0 <0.1.0")
- Do ..AssertEquivalent("^1.x", ">=1.0.0 <2.0.0")
- Do ..AssertEquivalent("^0.x", ">=0.0.0 <1.0.0")
+ Do ..AssertEquivalent("^1.2.3", ">=1.2.3 <2.0.0-0")
+ Do ..AssertEquivalent("^0.2.3", ">=0.2.3 <0.3.0-0")
+ Do ..AssertEquivalent("^0.0.3", ">=0.0.3 <0.0.4-0")
+ Do ..AssertEquivalent("^1.2.3-beta.2", ">=1.2.3-beta.2 <2.0.0-0")
+ Do ..AssertEquivalent("^0.0.3-beta", ">=0.0.3-beta <0.0.4-0")
+ Do ..AssertEquivalent("^1.2.x", ">=1.2.0 <2.0.0-0")
+ Do ..AssertEquivalent("^0.0.x", ">=0.0.0 <0.1.0-0")
+ Do ..AssertEquivalent("^0.0", ">=0.0.0 <0.1.0-0")
+ Do ..AssertEquivalent("^1.x", ">=1.0.0 <2.0.0-0")
+ Do ..AssertEquivalent("^0.x", ">=0.0.0 <1.0.0-0")
}
Method TestIRISVersions()
From 3b85a525327716b74e9409b2547c27d6dcee0cf6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 29 Aug 2024 14:00:26 -0400
Subject: [PATCH 036/182] feat: specify prerelease and snapshots when searching
for packages
---
src/cls/IPM/Repo/Filesystem/PackageService.cls | 1 +
src/cls/IPM/Repo/Remote/PackageService.cls | 2 ++
2 files changed, 3 insertions(+)
diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls
index a406c6c7..14c108f6 100644
--- a/src/cls/IPM/Repo/Filesystem/PackageService.cls
+++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls
@@ -12,6 +12,7 @@ Method %OnNew(pRoot As %String) As %Status [ Private, ServerOnly = 1 ]
Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObjects(ELEMENTTYPE="%IPM.Storage.ModuleInfo")
{
Set tList = ##class(%Library.ListOfObjects).%New()
+ // TODO: filter by pSearchCriteria.IncludeSnapshots and pSearchCriteria.IncludePrerelease
Set tQuery = "select Name,VersionString from %IPM_Repo_Filesystem.Cache_OrderedMatches(?,?,?,?) m"
Set tArgs($i(tArgs)) = ..Root
Quit ##class(%IPM.Repo.Utils).moduleSqlToList(tQuery,.pSearchCriteria,.tArgs)
diff --git a/src/cls/IPM/Repo/Remote/PackageService.cls b/src/cls/IPM/Repo/Remote/PackageService.cls
index b952a9de..f575fdf8 100644
--- a/src/cls/IPM/Repo/Remote/PackageService.cls
+++ b/src/cls/IPM/Repo/Remote/PackageService.cls
@@ -50,6 +50,8 @@ Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObject
Set tURL = tRequest.Location_"packages/" _ name
}
Do tRequest.SetParam("allVersions", pSearchCriteria.AllVersions)
+ Do tRequest.SetParam("includePrerelease", pSearchCriteria.IncludePrerelease)
+ Do tRequest.SetParam("includeSnapshots", pSearchCriteria.IncludeSnapshots)
Set tSC = tRequest.Get($$$URLENCODE(tURL))
From 5be72d77e1436f2754ad3e06003a828b462ab8f6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 29 Aug 2024 14:39:15 -0400
Subject: [PATCH 037/182] chore: remove unnecessary docstring
Apparently %IPM.Repo.Utils:moduleSqlToList() takes care of prerelease and snapshot already
---
src/cls/IPM/Repo/Filesystem/PackageService.cls | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Filesystem/PackageService.cls b/src/cls/IPM/Repo/Filesystem/PackageService.cls
index 14c108f6..a406c6c7 100644
--- a/src/cls/IPM/Repo/Filesystem/PackageService.cls
+++ b/src/cls/IPM/Repo/Filesystem/PackageService.cls
@@ -12,7 +12,6 @@ Method %OnNew(pRoot As %String) As %Status [ Private, ServerOnly = 1 ]
Method ListModules(pSearchCriteria As %IPM.Repo.SearchCriteria) As %ListOfObjects(ELEMENTTYPE="%IPM.Storage.ModuleInfo")
{
Set tList = ##class(%Library.ListOfObjects).%New()
- // TODO: filter by pSearchCriteria.IncludeSnapshots and pSearchCriteria.IncludePrerelease
Set tQuery = "select Name,VersionString from %IPM_Repo_Filesystem.Cache_OrderedMatches(?,?,?,?) m"
Set tArgs($i(tArgs)) = ..Root
Quit ##class(%IPM.Repo.Utils).moduleSqlToList(tQuery,.pSearchCriteria,.tArgs)
From 8caee9c9d8995d23f8a6cb14d0fa8e7f8cf44e8a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 30 Aug 2024 09:53:10 -0400
Subject: [PATCH 038/182] chore: update change log
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c111b5fb..f2dc9bf2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #224: When updating zpm, existing configuration won't be reset
- #482: Reenabled deployed code support without impact on embedded source control (reworks HSIEO-9277)
- #487: When loading a package, relative paths staring with prefix "http" won't be mistaken for git repo
+- #557: When comparing semver against semver expressions, exclude prereleases and snapshots from the range maximum.
### Security
-
From 4e0eaa0250ce51f896a4841d5a3ecf8c4a840ed5 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 30 Aug 2024 09:55:10 -0400
Subject: [PATCH 039/182] docs: update changelog
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c111b5fb..fbf971fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -51,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #224: When updating zpm, existing configuration won't be reset
- #482: Reenabled deployed code support without impact on embedded source control (reworks HSIEO-9277)
- #487: When loading a package, relative paths staring with prefix "http" won't be mistaken for git repo
+- #544: When installing a package from remote repo, IPM specifies `includePrerelease` and `includeSnapshots` in HTTP request. Correctly-behaving zpm registry should respect that.
### Security
-
From e7757d815b78fb107ed3ee102d05dc66e9b1c87f Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Sep 2024 10:13:35 -0400
Subject: [PATCH 040/182] feat: add global config SemVerPostRelease to treat
pre-rel. as post-rel.
---
preload/cls/IPM/Installer.cls | 1 +
src/cls/IPM/Repo/UniversalSettings.cls | 7 ++++++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index c7935511..b1be7fa9 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -104,6 +104,7 @@ ClassMethod ZPMInit(pRegistry As %String = "", pAnalyticsTrackingID As %String =
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("ColorScheme","", 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("PipCaller", "", 0))
$$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("UseStandalonePip", "", 0))
+ $$$QuitOnError(##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0, 0))
Quit $$$OK
}
diff --git a/src/cls/IPM/Repo/UniversalSettings.cls b/src/cls/IPM/Repo/UniversalSettings.cls
index b360a19e..ac5d75d8 100644
--- a/src/cls/IPM/Repo/UniversalSettings.cls
+++ b/src/cls/IPM/Repo/UniversalSettings.cls
@@ -34,7 +34,12 @@ Parameter PipCaller = "PipCaller";
/// Indicates whether PipCaller is a pip executable instead of python
Parameter UseStandalonePip = "UseStandalonePip";
-Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip";
+/// Possible values: 0, 1
+/// Indicates whether the SemVer comparison treats the version 1.0.0-1.m1 as a post-release of 1.0.0, hence 1.0.0-1.m1 > 1.0.0
+/// Default value is 0, where 1.0.0-anystring is considered a pre-release of 1.0.0, hence 1.0.0-anystring < 1.0.0
+Parameter SemVerPostRelease = "SemVerPostRelease";
+
+Parameter CONFIGURABLE = "trackingId,analytics,ColorScheme,TerminalPrompt,PublishTimeout,PipCaller,UseStandalonePip,SemVerPostRelease";
/// Returns configArray, that includes all configurable settings
ClassMethod GetAll(Output configArray) As %Status
From 757885339419762f8306226c5547eb55b7d3d464 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Sep 2024 14:21:18 -0400
Subject: [PATCH 041/182] feat(semver): support post-release comparison
---
src/cls/IPM/General/SemanticVersion.cls | 92 ++++++++++++-------
.../SemanticVersionExpression/Comparator.cls | 5 +-
2 files changed, 62 insertions(+), 35 deletions(-)
diff --git a/src/cls/IPM/General/SemanticVersion.cls b/src/cls/IPM/General/SemanticVersion.cls
index 38c7d99a..3b92fad7 100644
--- a/src/cls/IPM/General/SemanticVersion.cls
+++ b/src/cls/IPM/General/SemanticVersion.cls
@@ -13,8 +13,16 @@ Property Patch As %Integer(MINVAL = 0) [ Required ];
Property Prerelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
+/// This is an alias for Prerelease. It is used for code readability when SemVerPostRelease is enabled.
+Property Postrelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*") [ Calculated, SqlComputeCode = { Set {*} = Prerelease }, SqlComputed, Transient ];
+
Property Build As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
+Method PostreleaseGet() As %IPM.DataType.RegExString
+{
+ Quit ..Prerelease
+}
+
Method ToString() As %String [ CodeMode = expression ]
{
..Major
@@ -116,6 +124,25 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
((..Major = pVersion.Major) && (..Minor > pVersion.Minor)) ||
((..Major = pVersion.Major) && (..Minor = pVersion.Minor) && (..Patch > pVersion.Patch))
+ // Handle post-releases if enabled. With this, post-releases are considered as higher version numbers.
+ If ##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") {
+ If tFollows {
+ Return 1
+ }
+ If (..Major < pVersion.Major) || (..Minor < pVersion.Minor) || (..Patch < pVersion.Patch) {
+ Return 0
+ }
+ // If it reaches here, major, minor, and patch are equal. We need to check post-releases.
+ If (..Postrelease = "") {
+ Return 0
+ } ElseIf (pVersion.Postrelease = "") {
+ Return 1
+ }
+ // If it reaches here, major, minor, and patch are equal. Both post-releases are non-empty. Compare them.
+ Return ..CompareDotSepartedStrings(..Postrelease, pVersion.Postrelease)
+
+ }
+
// Handle prereleases - messy!!
Set tEquals = (..Major = pVersion.Major) && (..Minor = pVersion.Minor) && (..Patch = pVersion.Patch)
If (..Prerelease '= "") || (pVersion.Prerelease '= "") {
@@ -135,39 +162,7 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
// We are comparing equal versions where the earlier has a prerelease.
Quit 1
} Else{
- // Both have a prerelease, and they're different.
- // Compare dot-separated parts of the prerelease.
- Set tFollows = 1
- Set tThisParts = $ListFromString(..Prerelease,".")
- Set tOtherParts = $ListFromString(pVersion.Prerelease,".")
-
- Set tOtherHasData = 1
- Set tThisPointer = 0
- Set tOtherPointer = 0
- While $ListNext(tThisParts,tThisPointer,tThisPart) {
- Set tOtherHasData = $ListNext(tOtherParts,tOtherPointer,tOtherPart)
- If 'tOtherHasData {
- // The prerelease version has more parts for this one.
- Return 1
- }
- If (tOtherPart = tThisPart) {
- // Keep looking through dot-separated parts.
- Continue
- }
-
- // "Collates after" operator works nicely here.
- // e.g., the following are true: "beta" ]] 11, 11 ]] 2, 2 ]] 1
- If (tThisPart ]] tOtherPart) {
- Return 1
- } Else {
- Return 0
- }
- }
- If tFollows && tOtherHasData && $ListNext(tOtherParts,tOtherPointer,tOtherPart) {
- // If there are still dot-separated parts left in the prerelease of the version we are comparing to,
- // it has more than this version, and therefore this version does not follow it.
- Quit 0
- }
+ Quit ..CompareDotSepartedStrings(..Prerelease, pVersion.Prerelease)
}
} ElseIf tEquals {
Quit (pVersion.IsSnapshot() && '..IsSnapshot())
@@ -175,6 +170,37 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
Quit tFollows
}
+/// Compare two dot-separated strings (usually prerelease or postrelease identifiers).
+Method CompareDotSepartedStrings(pThis As %String, pOther As %String) As %Boolean
+{
+ Set tThisParts = $ListFromString(pThis,".")
+ Set tOtherParts = $ListFromString(pOther,".")
+
+ Set tOtherHasData = 1
+ Set tThisPointer = 0
+ Set tOtherPointer = 0
+ While $ListNext(tThisParts,tThisPointer,tThisPart) {
+ Set tOtherHasData = $ListNext(tOtherParts,tOtherPointer,tOtherPart)
+ If 'tOtherHasData {
+ // The prerelease version has more parts for this one.
+ Return 1
+ }
+ If (tOtherPart = tThisPart) {
+ // Keep looking through dot-separated parts.
+ Continue
+ }
+
+ // "Collates after" operator works nicely here.
+ // e.g., the following are true: "beta" ]] 11, 11 ]] 2, 2 ]] 1
+ If (tThisPart ]] tOtherPart) {
+ Return 1
+ } Else {
+ Return 0
+ }
+ }
+ Return 0
+}
+
Method Satisfies(pExpression As %IPM.General.SemanticVersionExpression) As %Boolean
{
Quit pExpression.IsSatisfiedBy($this)
diff --git a/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls b/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
index 2c9ea0ab..a56c44a5 100644
--- a/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
+++ b/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
@@ -93,8 +93,9 @@ Method Evaluate(pVersion As %IPM.General.SemanticVersion) As %Boolean
Set tEquals = tEquals && (val1 = val2)
}
}
- If ($Extract(..Operator,1) = "=") {
- // Means direct equality and not <= or >= so compare pre-release as well
+ If (##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") = 0) || ($Extract(..Operator,1) = "=") {
+ // If SemVerPostRelease is disabled, then we should not consider post-release versions as equal to the base version
+ // ..Operator = "=" means direct equality and not <= or >= so compare pre-release as well
Set tEquals = tEquals && (pVersion.Prerelease = ..Prerelease)
}
If tEquals || (..Operator = "=") {
From bf5cd99846ff7d1e2b228852847ef11ea3e5e942 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Sep 2024 14:21:58 -0400
Subject: [PATCH 042/182] test(semver): add test cases for post-release
comparisons
---
.../Test/PM/Unit/SemVer/Abstract.cls | 20 ++++++++++
.../Test/PM/Unit/SemVer/Expressions.cls | 36 ++++++++++++++++-
.../Test/PM/Unit/SemVer/Versions.cls | 39 ++++++++++++++++++-
3 files changed, 92 insertions(+), 3 deletions(-)
create mode 100644 tests/unit_tests/Test/PM/Unit/SemVer/Abstract.cls
diff --git a/tests/unit_tests/Test/PM/Unit/SemVer/Abstract.cls b/tests/unit_tests/Test/PM/Unit/SemVer/Abstract.cls
new file mode 100644
index 00000000..d250cd78
--- /dev/null
+++ b/tests/unit_tests/Test/PM/Unit/SemVer/Abstract.cls
@@ -0,0 +1,20 @@
+Class Test.PM.Unit.SemVer.Abstract Extends %UnitTest.TestCase
+{
+
+/// Original value of the SemVerPostRelease setting
+Property OriginalSemVerPostRelease As %Boolean [ InitialExpression = {##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease")} ];
+
+/// Before each test, record the original value of the SemVerPostRelease setting and set it to 0 for a clean slate
+Method OnBeforeOneTest(testname As %String) As %Status
+{
+ Set ..OriginalSemVerPostRelease = ##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease")
+ Return ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 0)
+}
+
+/// After each test, restore the original value of the SemVerPostRelease setting
+Method OnAfterOneTest(testname As %String) As %Status
+{
+ Return ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", ..OriginalSemVerPostRelease)
+}
+
+}
diff --git a/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls b/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
index dddddefc..a5c341fa 100644
--- a/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
+++ b/tests/unit_tests/Test/PM/Unit/SemVer/Expressions.cls
@@ -1,4 +1,4 @@
-Class Test.PM.Unit.SemVer.Expressions Extends %UnitTest.TestCase
+Class Test.PM.Unit.SemVer.Expressions Extends Test.PM.Unit.SemVer.Abstract
{
Method TestEvaluate()
@@ -29,12 +29,13 @@ Method TestEvaluate()
Method TestPrerelease()
{
+ Do ..AssertNotSatisfied(">=1.0.0", "1.0.0-1.m1")
Do ..AssertSatisfied("1.0.0-beta.1","1.0.0-beta.1")
Do ..AssertSatisfied("=1.0.0-beta.1","1.0.0-beta.1")
Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.1")
Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.2")
Do ..AssertSatisfied(">=1.0.0-alpha.1","1.0.0-beta.1")
- Do ..AssertNotSatisfied("^1.0.0","1.0.0-1.m1") // This shouldn't be satisfied because 1.0.0-1.m1 is a prerelease
+ Do ..AssertNotSatisfied("^1.0.0","1.0.0-1.m1")
Do ..AssertNotSatisfied(">=1.0.0-beta.2","1.0.0-beta.1")
Do ..AssertNotSatisfied(">=1.0.0-beta.1","1.0.0-alpha.1")
@@ -55,6 +56,37 @@ Method TestPrerelease()
Do ..AssertNotSatisfied("1.1.x", "1.2.0-alpha")
}
+Method TestPostRelease()
+{
+ // With SemVerPostRelease enabled, post-release versions should be considered as a higher version number
+ Set tSC = ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 1)
+ Do ..AssertSatisfied(">=1.0.0", "1.0.0-1.m1")
+ Do ..AssertSatisfied("1.0.0-beta.1","1.0.0-beta.1")
+ Do ..AssertSatisfied("=1.0.0-beta.1","1.0.0-beta.1")
+ Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.1")
+ Do ..AssertSatisfied(">=1.0.0-beta.1","1.0.0-beta.2")
+ Do ..AssertSatisfied(">=1.0.0-alpha.1","1.0.0-beta.1")
+
+ Do ..AssertSatisfied("^1.0.0","1.0.0-1.m1")
+ Do ..AssertNotSatisfied(">=1.0.0-beta.2","1.0.0-beta.1")
+ Do ..AssertNotSatisfied(">=1.0.0-beta.1","1.0.0-alpha.1")
+
+ // This is the problematic case without HSIEO-10520
+ Do ..AssertNotSatisfied("1.0.0-1.m1","1.0.0")
+ Do ..AssertNotSatisfied("=1.0.0-1.m1","1.0.0")
+
+ // With SemVerPostRelease enabled, 1.5.0-beta.1 is within the 1.x range, because it's a post-release of 1.5.0.
+ // Similarly, 1.1.5-beta.1 satisifies 1.1.x.
+ Do ..AssertSatisfied("1.x", "1.5.0-beta.1")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-0")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-SNAPSHOT")
+ Do ..AssertNotSatisfied("1.x", "2.0.0-alpha")
+ Do ..AssertSatisfied("1.1.x", "1.1.5-beta.1")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-0")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-SNAPSHOT")
+ Do ..AssertNotSatisfied("1.1.x", "1.2.0-alpha")
+}
+
Method TestEquivalenceHyphenRanges()
{
Do ..AssertEquivalent("1.2.3 - 2.3.4", ">=1.2.3 <=2.3.4")
diff --git a/tests/unit_tests/Test/PM/Unit/SemVer/Versions.cls b/tests/unit_tests/Test/PM/Unit/SemVer/Versions.cls
index f7f1c84b..3fa6c4ef 100644
--- a/tests/unit_tests/Test/PM/Unit/SemVer/Versions.cls
+++ b/tests/unit_tests/Test/PM/Unit/SemVer/Versions.cls
@@ -1,6 +1,29 @@
-Class Test.PM.Unit.SemVer.Versions Extends %UnitTest.TestCase
+Class Test.PM.Unit.SemVer.Versions Extends Test.PM.Unit.SemVer.Abstract
{
+Method TestPrereleaseComparison()
+{
+ Do ..AssertFollows("1.0.0", "1.0.0-beta")
+ Do ..AssertFollows("1.0.0-beta", "1.0.0-alpha.1")
+ Do ..AssertFollows("1.0.0-alpha.1", "1.0.0-alpha")
+ Do ..AssertFollows("2.0.0", "1.0.0-alpha")
+
+ Do ..AssertNotFollows("1.0.0-alpha", "1.0.0-alpha.1")
+ Do ..AssertNotFollows("1.0.0-alpha", "1.0.0-alpha.beta")
+ Do ..AssertNotFollows("1.0.0-beta", "1.0.0")
+ Do ..AssertNotFollows("2.0.0-alpha", "1.0.0-alpha")
+}
+
+Method TestPostreleaseComparison()
+{
+ Do ##class(%IPM.Repo.UniversalSettings).SetValue("SemVerPostRelease", 1)
+ Do ..AssertFollows("1.0.0-1.m1", "1.0.0")
+ Do ..AssertFollows("1.0.0-1.m2", "1.0.0-1.m1")
+
+ // Regular release should not follow post-release (we are treating -alpha as a post-release)
+ Do ..AssertNotFollows("1.0.0", "1.0.0-alpha")
+}
+
Method TestVersions()
{
Do ..AssertVersionValid("1.0.0-alpha", 1, 0, 0, "alpha")
@@ -43,4 +66,18 @@ Method AssertVersionNotValid(pVersion As %String)
Do $$$AssertStatusNotOK(##class(%IPM.General.SemanticVersion).IsValid(pVersion),pVersion_" is not accepted as a valid semantic version.")
}
+Method AssertFollows(pVersion1 As %String, pVersion2 As %String)
+{
+ Set tSemVer1 = ##class(%IPM.General.SemanticVersion).FromString(pVersion1)
+ Set tSemVer2 = ##class(%IPM.General.SemanticVersion).FromString(pVersion2)
+ Do $$$AssertEquals(tSemVer1.Follows(tSemVer2), 1, pVersion1_" follows "_pVersion2)
+}
+
+Method AssertNotFollows(pVersion1 As %String, pVersion2 As %String)
+{
+ Set tSemVer1 = ##class(%IPM.General.SemanticVersion).FromString(pVersion1)
+ Set tSemVer2 = ##class(%IPM.General.SemanticVersion).FromString(pVersion2)
+ Do $$$AssertEquals(tSemVer1.Follows(tSemVer2), 0, pVersion1_" does not follow "_pVersion2)
+}
+
}
From 934a6e1fe5db83b1e1bfdca22510acf88b215e4c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Sep 2024 14:25:06 -0400
Subject: [PATCH 043/182] chore(changelog): explain post release feature
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f2dc9bf2..90f41411 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -52,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #482: Reenabled deployed code support without impact on embedded source control (reworks HSIEO-9277)
- #487: When loading a package, relative paths staring with prefix "http" won't be mistaken for git repo
- #557: When comparing semver against semver expressions, exclude prereleases and snapshots from the range maximum.
+- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
### Security
-
From 002999146238527ca98d8bf302c3badba2649bcb Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Sep 2024 14:44:41 -0400
Subject: [PATCH 044/182] fix: coerce SemVerPostRelease to 0 in case installer
is not run
---
src/cls/IPM/General/SemanticVersionExpression/Comparator.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls b/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
index a56c44a5..05312cd1 100644
--- a/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
+++ b/src/cls/IPM/General/SemanticVersionExpression/Comparator.cls
@@ -93,7 +93,7 @@ Method Evaluate(pVersion As %IPM.General.SemanticVersion) As %Boolean
Set tEquals = tEquals && (val1 = val2)
}
}
- If (##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") = 0) || ($Extract(..Operator,1) = "=") {
+ If (+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease") = 0) || ($Extract(..Operator,1) = "=") {
// If SemVerPostRelease is disabled, then we should not consider post-release versions as equal to the base version
// ..Operator = "=" means direct equality and not <= or >= so compare pre-release as well
Set tEquals = tEquals && (pVersion.Prerelease = ..Prerelease)
From 627fcb524eaf6dcc4948c779379cf642e567cfe3 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 4 Sep 2024 09:35:09 -0400
Subject: [PATCH 045/182] style: fix misspelt method name
---
src/cls/IPM/General/SemanticVersion.cls | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/General/SemanticVersion.cls b/src/cls/IPM/General/SemanticVersion.cls
index 3b92fad7..1c32886a 100644
--- a/src/cls/IPM/General/SemanticVersion.cls
+++ b/src/cls/IPM/General/SemanticVersion.cls
@@ -139,7 +139,7 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
Return 1
}
// If it reaches here, major, minor, and patch are equal. Both post-releases are non-empty. Compare them.
- Return ..CompareDotSepartedStrings(..Postrelease, pVersion.Postrelease)
+ Return ..CompareDotSeparatedStrings(..Postrelease, pVersion.Postrelease)
}
@@ -162,7 +162,7 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
// We are comparing equal versions where the earlier has a prerelease.
Quit 1
} Else{
- Quit ..CompareDotSepartedStrings(..Prerelease, pVersion.Prerelease)
+ Quit ..CompareDotSeparatedStrings(..Prerelease, pVersion.Prerelease)
}
} ElseIf tEquals {
Quit (pVersion.IsSnapshot() && '..IsSnapshot())
@@ -171,7 +171,7 @@ Method Follows(pVersion As %IPM.General.SemanticVersion) As %Boolean
}
/// Compare two dot-separated strings (usually prerelease or postrelease identifiers).
-Method CompareDotSepartedStrings(pThis As %String, pOther As %String) As %Boolean
+Method CompareDotSeparatedStrings(pThis As %String, pOther As %String) As %Boolean
{
Set tThisParts = $ListFromString(pThis,".")
Set tOtherParts = $ListFromString(pOther,".")
From 5e7fbb160de1f64c6cfeda9cee364ae8ab0052ec Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 4 Sep 2024 09:46:50 -0400
Subject: [PATCH 046/182] fix(ci): use non-prerelease version of 0.7 for new
version of registry
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b526fc9e..64332c5a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -169,7 +169,7 @@ jobs:
run: |
curl http://localhost:52773/registry/packages/-/all | jq
curl http://localhost:52773/registry/packages/zpm/ | jq
- ASSET_NAME='zpm-0.7.1-beta.4.xml'
+ ASSET_NAME='zpm-0.7.2.xml'
ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
wget $ASSET_URL -O /tmp/zpm.xml
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
From b9552ee494415a3879236e9bcbed82ad58e3e0db Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 12:32:01 -0400
Subject: [PATCH 047/182] feat: adjust sort order when using post-release
---
src/cls/IPM/Repo/Manager.cls | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 291d6ff5..5902a20d 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -114,7 +114,8 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
Quit:patch=""
Set sub = ""
For {
- Set sub = $Order(versions(major, minor, patch, sub), 1)
+ Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
+ Set sub = $Order(versions(major, minor, patch, sub), subDirection)
Quit:sub=""
Set repoOrder = ""
For {
From d7447481fadca9ab60bc909bdcad15583f0a4f2c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 12:32:01 -0400
Subject: [PATCH 048/182] feat: adjust sort order when using post-release
---
src/cls/IPM/Repo/Manager.cls | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 291d6ff5..5902a20d 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -114,7 +114,8 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
Quit:patch=""
Set sub = ""
For {
- Set sub = $Order(versions(major, minor, patch, sub), 1)
+ Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
+ Set sub = $Order(versions(major, minor, patch, sub), subDirection)
Quit:sub=""
Set repoOrder = ""
For {
From 4fe93007ac3925a0487a238c6cc969a6444b7c9f Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 12:58:22 -0400
Subject: [PATCH 049/182] Revert "feat: adjust sort order when using
post-release"
This reverts commit b9552ee494415a3879236e9bcbed82ad58e3e0db.
---
src/cls/IPM/Repo/Manager.cls | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 5902a20d..291d6ff5 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -114,8 +114,7 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
Quit:patch=""
Set sub = ""
For {
- Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
- Set sub = $Order(versions(major, minor, patch, sub), subDirection)
+ Set sub = $Order(versions(major, minor, patch, sub), 1)
Quit:sub=""
Set repoOrder = ""
For {
From a4b2cb8d2f64eac71998756024403543409cd737 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 13:04:45 -0400
Subject: [PATCH 050/182] enhance: set version sort direction outside of for
loop
---
src/cls/IPM/Repo/Manager.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 5902a20d..139561d4 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -113,8 +113,8 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
Set patch = $Order(versions(major, minor, patch), -1)
Quit:patch=""
Set sub = ""
+ Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
For {
- Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
Set sub = $Order(versions(major, minor, patch, sub), subDirection)
Quit:sub=""
Set repoOrder = ""
From 96fc4a2544de80dd55c40a27d599097c119ecade Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 13:06:58 -0400
Subject: [PATCH 051/182] enhance: move sorting direction further outside the
loops
---
src/cls/IPM/Repo/Manager.cls | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 139561d4..8de624ae 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -101,6 +101,8 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
}
}
Set major = ""
+ // Sorting direction for prereleases/postreleases. Only set once outside of the whole loop.
+ Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
For {
Set major = $Order(versions(major), -1)
Quit:major=""
@@ -113,7 +115,6 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
Set patch = $Order(versions(major, minor, patch), -1)
Quit:patch=""
Set sub = ""
- Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
For {
Set sub = $Order(versions(major, minor, patch, sub), subDirection)
Quit:sub=""
From 5f53f89db2f09f3f92aab899568faf106d1a0342 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 13:08:54 -0400
Subject: [PATCH 052/182] style: indentation
At some point we should really enforce a tab/space style guide and make it part of .vscode/settings.json
---
src/cls/IPM/Repo/Manager.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Repo/Manager.cls b/src/cls/IPM/Repo/Manager.cls
index 8de624ae..c909f6b8 100644
--- a/src/cls/IPM/Repo/Manager.cls
+++ b/src/cls/IPM/Repo/Manager.cls
@@ -101,7 +101,7 @@ Method SearchRepositoriesForModule(pSearchCriteria As %IPM.Repo.SearchCriteria,
}
}
Set major = ""
- // Sorting direction for prereleases/postreleases. Only set once outside of the whole loop.
+ // Sorting direction for prereleases/postreleases. Only set once outside of the whole loop.
Set subDirection = $SELECT(+##class(%IPM.Repo.UniversalSettings).GetValue("SemVerPostRelease"): -1, 1: 1)
For {
Set major = $Order(versions(major), -1)
From e98029a53cd56c4be8b0c8c0c21fe343d616cddd Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Sep 2024 14:22:00 -0400
Subject: [PATCH 053/182] feat: implement generic webapp processor
---
.../IPM/ResourceProcessor/WebApplication.cls | 243 ++++++++++++++++++
1 file changed, 243 insertions(+)
create mode 100644 src/cls/IPM/ResourceProcessor/WebApplication.cls
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
new file mode 100644
index 00000000..a2f2230e
--- /dev/null
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -0,0 +1,243 @@
+Include (%sySecurity, %occErrors, %occReference)
+
+Class %IPM.ResourceProcessor.WebApplication Extends (%IPM.ResourceProcessor.Abstract, %XML.Adaptor) [ PropertyClass = %IPM.ResourceProcessor.PropertyParameters ]
+{
+
+/// Description of resource processor class (shown in UI)
+Parameter DESCRIPTION As STRING = "Facilitates Web Application create/update/delete.";
+
+/// Name of the web application.
+Property Name As %String [ Calculated ];
+
+Method NameGet() As %String
+{
+ Set tName = $Get(..SecurityProperties("Name"), $Get(..SecurityProperties("Url"), ""))
+ If tName = "" {
+ Throw $$$ERROR($$$GeneralError, "Name is required for web applications")
+ }
+ Return tName
+}
+
+/// Keeps track of all application properties
+Property SecurityProperties As %RawString [ MultiDimensional ];
+
+Method CopyAttributes() [ Private ]
+{
+ /*
+ * Overridden to avoid needing a property in this class to correspond to each
+ * attribute as the attribute list is just all non-private/transient properties
+ * in Security.Applications.
+ */
+ Set key = ""
+ While 1 {
+ Set value = ..ResourceReference.Attributes.GetNext(.key)
+ If (key = "") {
+ Quit
+ }
+ Set eval = ..%Evaluate(value)
+ Set ..SecurityProperties(key) = eval
+ }
+}
+
+Method ReplaceMatchRoles(matchRoles As %String, dbDir As %String) As %String
+{
+ New $NAMESPACE
+ Set $NAMESPACE = "%SYS"
+ Set templates = $Listbuild("{$dbrole}", "${dbrole}")
+ For i=1:1:$Listlength(templates) {
+ Set template = $Listget(templates, i)
+ If matchRoles[template {
+ Set dbRole = "%DB_DEFAULT"
+ Set db = ##class(SYS.Database).%OpenId(dbDir)
+ If $Isobject(db) {
+ Set dbRole = db.ResourceName
+ }
+ Set matchRoles = $Replace(matchRoles, template, dbRole)
+ }
+ }
+ Return matchRoles
+}
+
+/// Called as phase pPhase is executed for the resource. If pResourceHandled is set to true,
+/// then the default behavior for that resource will be bypassed in the current phase.
+Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boolean = 0) As %Status
+{
+ Set sc = $$$OK
+ Try {
+ If (pPhase = "Validate") {
+ If (..Name = "") {
+ Return $$$ERROR($$$GeneralError,"Name is required for web applications")
+ }
+ Set keyList = ""
+ Set key = ""
+ While 1 {
+ Set value = ..ResourceReference.Attributes.GetNext(.key)
+ If (key = "") {
+ Quit
+ }
+ Set keyList = keyList _ $ListBuild(key)
+ }
+ Set badKeyList = ..DoPropertiesExist(keyList)
+ If ($ListLength(badKeyList) > 0) {
+ Set error = "The following invalid attributes were found for resource "_..ResourceReference.Name_": "_$ListToString(badKeyList)
+ $$$ThrowStatus($$$ERROR($$$GeneralError,error))
+ }
+ }
+ } Catch (ex) {
+ Set sc = ex.AsStatus()
+ }
+ Return sc
+}
+
+Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
+{
+ /*
+ * NOTE: Web app creation is done in OnBeforePhase so it happens before other
+ * resource processors such as FileCopy run.
+ */
+ Set sc = $$$OK
+ Try {
+ $$$ThrowOnError(##super(pPhase,.pParams))
+
+ Set tVerbose = $Get(pParams("Verbose"))
+ #; Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%ZHSLIB.PackageManager.Developer.Lifecycle.Application).%ClassName(1))
+ Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%IPM.Lifecycle.DeployedModule).%ClassName(1))
+
+ // TODO: Make mirror-safe?
+ // Would require moving mirror-safe APIs to the package manager - at which point, why not just have the package manager manage the whole federation?
+ If ((pPhase = "Configure") || ((pPhase = "Activate") && tIsApplication)) {
+ // Create web application
+ Do ..CreateOrUpdateWebApp(tVerbose)
+ } ElseIf ((pPhase = "Unconfigure") || ((pPhase = "Clean") && tIsApplication)) {
+ // Remove web application
+ Do ..DeleteWebApp(tVerbose)
+ }
+ } Catch e {
+ Set sc = e.AsStatus()
+ }
+ Quit sc
+}
+
+Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
+{
+ // TODO: Make mirror-safe.
+ New $namespace
+ Set $namespace = "%SYS"
+
+ Merge properties = ..SecurityProperties
+ If $D(properties("MatchRoles"), tMatchRoles) # 2 {
+ Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, $$$defdir)
+ }
+
+ If ##class(Security.Applications).Exists(..Name) {
+ Write:pVerbose !,"Updating Web Application ",..Name
+ Set sc = ##class(Security.Applications).Get(..Name,.oldProperties)
+ $$$ThrowOnError(sc)
+
+ Kill changes
+ Set key = ""
+ For {
+ Set key = $Order(properties(key),1,value)
+ Quit:(key="")
+ Set oldValue = $Get(oldProperties(key))
+ If (value '= oldValue) {
+ If (value = "") {
+ Set value = "[missing]"
+ }
+ If (oldValue = "") {
+ Set oldValue = "[missing]"
+ }
+ Set changes($i(changes)) = key_": "_oldValue_" -> "_value
+ }
+ }
+
+ If $Data(changes) {
+ If (pVerbose) {
+ For i=1:1:$Get(changes) {
+ Write !,changes(i)
+ }
+ }
+ Set sc = ##class(Security.Applications).Modify(..Name,.properties)
+ $$$ThrowOnError(sc)
+ Write:pVerbose !,"Done."
+ } Else {
+ Write:pVerbose !,"No changes detected or made."
+ }
+ } Else {
+ Write:pVerbose !,"Creating Web Application ",..Name
+ Set sc = ##class(Security.Applications).Create(..Name,.properties)
+ If (pVerbose) {
+ Set key = ""
+ For {
+ Set key = $Order(properties(key),1,tValue)
+ Quit:key=""
+ Write !,key,": ",tValue
+ }
+ }
+ $$$ThrowOnError(sc)
+ Write:pVerbose !,"Done."
+ }
+}
+
+/// This removes an existing CSP application
+Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
+{
+ // TODO: Make mirror-safe.
+
+ // Only try to purge files if the directory exists.
+ Set sc = ##class(%Library.EnsembleMgr).deletePortal(..Name,"",pVerbose)
+ Set errorCodes = $System.Status.GetErrorCodes(sc)
+ If (errorCodes [ $$$ApplicationDoesNotExist || errorCodes [ $$$DeleteObjectNotFound) {
+ // Not actually a problem - allow Clean/Uninstall to continue if it fails while trying to remove something that doesn't exist.
+ Set sc = $$$OK
+ }
+ $$$ThrowOnError(sc)
+}
+
+/// Returns a unique name for this resource.
+/// Default implementation, leaving pUniqueName undefined, reverts to default behavior.
+/// An empty string indicates guaranteed uniqueness (useful for folders relative to module root)
+Method OnGetUniqueName(Output pUniqueName)
+{
+ If (..Name '= "") {
+ Set pUniqueName = ..Name
+ }
+}
+
+Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.ResourceInfo) As %Status
+{
+ Set pInfo = ##class(%IPM.ExtensionBase.SourceControl.ResourceInfo).%New()
+ If (..Name '= "") {
+ Set pInfo.SourceControlAware = 1
+ Set pInfo.ResourceType = "/CSP/"
+ Set pInfo.Prefix = ..Name
+ Set pInfo.RelativePath = ..ResourceReference.Name
+ Set pInfo.IsDirectory = ($Piece(..ResourceReference.Name,"/",*) '[ ".")
+ }
+ Quit $$$OK
+}
+
+/// Returns the path relative to the module root for item pItemName within this resource.
+Method OnItemRelativePath(pItemName As %String) As %String
+{
+ Quit ..ResourceReference.Name_$Piece(pItemName,..Name,2)
+}
+
+/// Check if the provided properties exist in Security.Applications.
+/// Returns a list of properties that do not exist.
+ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ Set badList = ""
+ Set ptr = 0
+ While $ListNext(pPropList,ptr,prop) {
+ If '$$$comMemberDefined("Security.Applications",$$$cCLASSproperty,prop) {
+ Set badList = badList _ $ListBuild(prop)
+ }
+ }
+ Return badList
+}
+
+}
From 69e8a5e32f113873619dcb960e910ded28e537b0 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 09:15:17 -0400
Subject: [PATCH 054/182] style: fix indent using spaces
---
.../IPM/ResourceProcessor/WebApplication.cls | 104 +++++++++---------
1 file changed, 52 insertions(+), 52 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index a2f2230e..40494d71 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -23,7 +23,7 @@ Property SecurityProperties As %RawString [ MultiDimensional ];
Method CopyAttributes() [ Private ]
{
- /*
+ /*
* Overridden to avoid needing a property in this class to correspond to each
* attribute as the attribute list is just all non-private/transient properties
* in Security.Applications.
@@ -43,19 +43,19 @@ Method ReplaceMatchRoles(matchRoles As %String, dbDir As %String) As %String
{
New $NAMESPACE
Set $NAMESPACE = "%SYS"
- Set templates = $Listbuild("{$dbrole}", "${dbrole}")
- For i=1:1:$Listlength(templates) {
- Set template = $Listget(templates, i)
- If matchRoles[template {
- Set dbRole = "%DB_DEFAULT"
- Set db = ##class(SYS.Database).%OpenId(dbDir)
- If $Isobject(db) {
- Set dbRole = db.ResourceName
- }
- Set matchRoles = $Replace(matchRoles, template, dbRole)
- }
- }
- Return matchRoles
+ Set templates = $Listbuild("{$dbrole}", "${dbrole}")
+ For i=1:1:$Listlength(templates) {
+ Set template = $Listget(templates, i)
+ If matchRoles[template {
+ Set dbRole = "%DB_DEFAULT"
+ Set db = ##class(SYS.Database).%OpenId(dbDir)
+ If $Isobject(db) {
+ Set dbRole = db.ResourceName
+ }
+ Set matchRoles = $Replace(matchRoles, template, dbRole)
+ }
+ }
+ Return matchRoles
}
/// Called as phase pPhase is executed for the resource. If pResourceHandled is set to true,
@@ -86,7 +86,7 @@ Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boo
} Catch (ex) {
Set sc = ex.AsStatus()
}
- Return sc
+ Return sc
}
Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
@@ -95,36 +95,36 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
* NOTE: Web app creation is done in OnBeforePhase so it happens before other
* resource processors such as FileCopy run.
*/
- Set sc = $$$OK
- Try {
- $$$ThrowOnError(##super(pPhase,.pParams))
-
- Set tVerbose = $Get(pParams("Verbose"))
- #; Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%ZHSLIB.PackageManager.Developer.Lifecycle.Application).%ClassName(1))
+ Set sc = $$$OK
+ Try {
+ $$$ThrowOnError(##super(pPhase,.pParams))
+
+ Set tVerbose = $Get(pParams("Verbose"))
+ #; Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%ZHSLIB.PackageManager.Developer.Lifecycle.Application).%ClassName(1))
Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%IPM.Lifecycle.DeployedModule).%ClassName(1))
-
- // TODO: Make mirror-safe?
- // Would require moving mirror-safe APIs to the package manager - at which point, why not just have the package manager manage the whole federation?
- If ((pPhase = "Configure") || ((pPhase = "Activate") && tIsApplication)) {
- // Create web application
- Do ..CreateOrUpdateWebApp(tVerbose)
- } ElseIf ((pPhase = "Unconfigure") || ((pPhase = "Clean") && tIsApplication)) {
- // Remove web application
- Do ..DeleteWebApp(tVerbose)
- }
- } Catch e {
- Set sc = e.AsStatus()
- }
- Quit sc
+
+ // TODO: Make mirror-safe?
+ // Would require moving mirror-safe APIs to the package manager - at which point, why not just have the package manager manage the whole federation?
+ If ((pPhase = "Configure") || ((pPhase = "Activate") && tIsApplication)) {
+ // Create web application
+ Do ..CreateOrUpdateWebApp(tVerbose)
+ } ElseIf ((pPhase = "Unconfigure") || ((pPhase = "Clean") && tIsApplication)) {
+ // Remove web application
+ Do ..DeleteWebApp(tVerbose)
+ }
+ } Catch e {
+ Set sc = e.AsStatus()
+ }
+ Quit sc
}
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
- // TODO: Make mirror-safe.
+ // TODO: Make mirror-safe.
New $namespace
Set $namespace = "%SYS"
- Merge properties = ..SecurityProperties
+ Merge properties = ..SecurityProperties
If $D(properties("MatchRoles"), tMatchRoles) # 2 {
Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, $$$defdir)
}
@@ -182,8 +182,8 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
/// This removes an existing CSP application
Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
- // TODO: Make mirror-safe.
-
+ // TODO: Make mirror-safe.
+
// Only try to purge files if the directory exists.
Set sc = ##class(%Library.EnsembleMgr).deletePortal(..Name,"",pVerbose)
Set errorCodes = $System.Status.GetErrorCodes(sc)
@@ -199,28 +199,28 @@ Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
/// An empty string indicates guaranteed uniqueness (useful for folders relative to module root)
Method OnGetUniqueName(Output pUniqueName)
{
- If (..Name '= "") {
- Set pUniqueName = ..Name
- }
+ If (..Name '= "") {
+ Set pUniqueName = ..Name
+ }
}
Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.ResourceInfo) As %Status
{
- Set pInfo = ##class(%IPM.ExtensionBase.SourceControl.ResourceInfo).%New()
- If (..Name '= "") {
- Set pInfo.SourceControlAware = 1
- Set pInfo.ResourceType = "/CSP/"
- Set pInfo.Prefix = ..Name
- Set pInfo.RelativePath = ..ResourceReference.Name
- Set pInfo.IsDirectory = ($Piece(..ResourceReference.Name,"/",*) '[ ".")
- }
- Quit $$$OK
+ Set pInfo = ##class(%IPM.ExtensionBase.SourceControl.ResourceInfo).%New()
+ If (..Name '= "") {
+ Set pInfo.SourceControlAware = 1
+ Set pInfo.ResourceType = "/CSP/"
+ Set pInfo.Prefix = ..Name
+ Set pInfo.RelativePath = ..ResourceReference.Name
+ Set pInfo.IsDirectory = ($Piece(..ResourceReference.Name,"/",*) '[ ".")
+ }
+ Quit $$$OK
}
/// Returns the path relative to the module root for item pItemName within this resource.
Method OnItemRelativePath(pItemName As %String) As %String
{
- Quit ..ResourceReference.Name_$Piece(pItemName,..Name,2)
+ Quit ..ResourceReference.Name_$Piece(pItemName,..Name,2)
}
/// Check if the provided properties exist in Security.Applications.
From 66cb10f24fc252ee0e4ac77ff959b088eb239407 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 09:25:51 -0400
Subject: [PATCH 055/182] test(processor): add WSGI test case for
WebApplication resource
---
.../PM/Integration/InstallApplication.cls | 18 ++++++++++
.../PM/Integration/_data/wsgi-app/module.xml | 33 +++++++++++++++++++
.../_data/wsgi-app/requirements.txt | 2 ++
.../_data/wsgi-app/src/python/flaskapp/app.py | 7 ++++
4 files changed, 60 insertions(+)
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/wsgi-app/requirements.txt
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/wsgi-app/src/python/flaskapp/app.py
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 49611609..7e5b33e5 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -29,4 +29,22 @@ Method TestSimpleApp()
}
}
+Method TestWSGIApp()
+{
+ Set tSC = $$$OK
+ Try {
+ Set tVersion = $Piece($ZVersion," ",9)
+ If $Piece(tVersion, ".", 1) < 2024 {
+ Do $$$AssertSkipped("WSGI applications are only supported in 2024+. Current version: "_tVersion)
+ Return
+ }
+ Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
+ set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/wsgi-app/")
+ Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
+ Do $$$AssertStatusOK(tSC,"Loaded WSGI Application successfully. " _ tModuleDir)
+ } Catch e {
+ Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
+ }
+}
+
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
new file mode 100644
index 00000000..9825dd59
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -0,0 +1,33 @@
+
+
+
+
+ flask-demo
+ 1.0.0
+ This is a demo of a flask application
+ flask
+
+ Shuheng Liu
+ InterSystems
+ 2024
+ MIT
+ notes
+
+ module
+
+ src
+
+
+
+ Module installed successfully!
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/requirements.txt b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/requirements.txt
new file mode 100644
index 00000000..55637ba5
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/requirements.txt
@@ -0,0 +1,2 @@
+# This is where you list all python dependencies, including Flask
+flask >= 3.0.0, < 4.0.0
\ No newline at end of file
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/src/python/flaskapp/app.py b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/src/python/flaskapp/app.py
new file mode 100644
index 00000000..e58e7abf
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/src/python/flaskapp/app.py
@@ -0,0 +1,7 @@
+from flask import Flask
+
+app = Flask(__name__)
+
+@app.route("/")
+def index():
+ return "This is a sample WSGI application using Flask!"
\ No newline at end of file
From 59bfc2dd7355b9c197895c540f25d48ea197fabd Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 10:08:26 -0400
Subject: [PATCH 056/182] test(processor): add REST test case for
WebApplication resource
---
.../PM/Integration/InstallApplication.cls | 13 +++++++
.../PM/Integration/_data/rest-app/module.xml | 35 +++++++++++++++++++
.../_data/rest-app/src/cls/Test/Rest/Demo.cls | 4 +++
3 files changed, 52 insertions(+)
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app/src/cls/Test/Rest/Demo.cls
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 7e5b33e5..8331c67c 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -47,4 +47,17 @@ Method TestWSGIApp()
}
}
+Method TestRestApp()
+{
+ Set tSC = $$$OK
+ Try {
+ Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
+ set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
+ Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
+ Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
+ } Catch e {
+ Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
+ }
+}
+
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
new file mode 100644
index 00000000..6ac103a8
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -0,0 +1,35 @@
+
+
+
+
+ rest-demo
+ 1.0.0
+ This is a demo of a flask application
+ flask
+
+ Shuheng Liu
+ InterSystems
+ 2024
+ MIT
+ notes
+
+ module
+ src
+
+
+ Module installed successfully!
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/src/cls/Test/Rest/Demo.cls b/tests/integration_tests/Test/PM/Integration/_data/rest-app/src/cls/Test/Rest/Demo.cls
new file mode 100644
index 00000000..8ae71916
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/src/cls/Test/Rest/Demo.cls
@@ -0,0 +1,4 @@
+Class Test.Rest.Demo Extends %CSP.REST
+{
+
+}
From f17681206cb17e9f0c028619799d06b387876ac2 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 10:54:48 -0400
Subject: [PATCH 057/182] fix(processor): set namespace for applications
---
.../IPM/ResourceProcessor/WebApplication.cls | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 40494d71..ebe68e0b 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -58,6 +58,18 @@ Method ReplaceMatchRoles(matchRoles As %String, dbDir As %String) As %String
Return matchRoles
}
+Method ReplaceNameSpace(expression As %String, namespace As %String) As %String
+{
+ Set templates = $Listbuild("{$ns}", "${ns}")
+ Set ptr = 0
+ While $LISTNEXT(templates, ptr, template) {
+ If expression[template {
+ Set expression = $REPLACE(expression, template, namespace)
+ }
+ }
+ Return expression
+}
+
/// Called as phase pPhase is executed for the resource. If pResourceHandled is set to true,
/// then the default behavior for that resource will be bypassed in the current phase.
Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boolean = 0) As %Status
@@ -121,6 +133,7 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
// TODO: Make mirror-safe.
+ Set tOriginalNS = $namespace
New $namespace
Set $namespace = "%SYS"
@@ -128,6 +141,12 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
If $D(properties("MatchRoles"), tMatchRoles) # 2 {
Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, $$$defdir)
}
+ If $D(properties("NameSpace"), tNameSpace) {
+ Set properties("NameSpace") = ..ReplaceNameSpace(tNameSpace, tOriginalNS)
+ } Else {
+ // Does WSGI application need a namespace?
+ Set properties("NameSpace") = tOriginalNS
+ }
If ##class(Security.Applications).Exists(..Name) {
Write:pVerbose !,"Updating Web Application ",..Name
From 959da4f68881f7ba66f2671e1ece786814d95506 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 11:37:30 -0400
Subject: [PATCH 058/182] fix(processor): evaluate $zu(12, "") in original ns
rather than %SYS
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index ebe68e0b..35ef56a5 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -134,12 +134,13 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
// TODO: Make mirror-safe.
Set tOriginalNS = $namespace
+ Set dbDir = $$$defdir
New $namespace
Set $namespace = "%SYS"
Merge properties = ..SecurityProperties
If $D(properties("MatchRoles"), tMatchRoles) # 2 {
- Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, $$$defdir)
+ Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, dbDir)
}
If $D(properties("NameSpace"), tNameSpace) {
Set properties("NameSpace") = ..ReplaceNameSpace(tNameSpace, tOriginalNS)
From c94bdd2fa2eeb8985544e85196848bca20836ca3 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 6 Sep 2024 11:39:30 -0400
Subject: [PATCH 059/182] style(indent): fix indent using spaces
---
.vscode/settings.json | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 6b0c540f..a79b849e 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,11 +1,8 @@
{
"objectscript.conn": {
"active": true,
- "port": 52774,
- "host": "localhost",
- "username": "_system",
- "password": "SYS",
- "ns": "USER"
+ "ns": "USER",
+ "server": "ipm09"
},
"intersystems.testingManager.client.relativeTestRoot": "tests/unit_tests"
}
\ No newline at end of file
From 98ba86f2dc85dc005eef132bd1dba67a8cf960cf Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 08:43:56 -0400
Subject: [PATCH 060/182] Revert "style(indent): fix indent using spaces"
This reverts commit c94bdd2fa2eeb8985544e85196848bca20836ca3.
Accidentally committed the wrong file
---
.vscode/settings.json | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json
index a79b849e..6b0c540f 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,8 +1,11 @@
{
"objectscript.conn": {
"active": true,
- "ns": "USER",
- "server": "ipm09"
+ "port": 52774,
+ "host": "localhost",
+ "username": "_system",
+ "password": "SYS",
+ "ns": "USER"
},
"intersystems.testingManager.client.relativeTestRoot": "tests/unit_tests"
}
\ No newline at end of file
From 2b0ad9a0f5e5afeb7b11ca4e3d10796de70b487e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 08:44:34 -0400
Subject: [PATCH 061/182] style(indent): fix indent using spaces
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 35ef56a5..7fe1425f 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -134,7 +134,7 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
// TODO: Make mirror-safe.
Set tOriginalNS = $namespace
- Set dbDir = $$$defdir
+ Set dbDir = $$$defdir
New $namespace
Set $namespace = "%SYS"
From 632709ea457857a666b690268f80d5977eb6a5e4 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:43:23 -0400
Subject: [PATCH 062/182] fix(webapp): fix operator precedence
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 7fe1425f..63be1429 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -207,7 +207,7 @@ Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
// Only try to purge files if the directory exists.
Set sc = ##class(%Library.EnsembleMgr).deletePortal(..Name,"",pVerbose)
Set errorCodes = $System.Status.GetErrorCodes(sc)
- If (errorCodes [ $$$ApplicationDoesNotExist || errorCodes [ $$$DeleteObjectNotFound) {
+ If ((errorCodes [ $$$ApplicationDoesNotExist) || (errorCodes [ $$$DeleteObjectNotFound)) {
// Not actually a problem - allow Clean/Uninstall to continue if it fails while trying to remove something that doesn't exist.
Set sc = $$$OK
}
From c56f3243fa0ccdc851ec3c50a6b734bb4050f73c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:44:13 -0400
Subject: [PATCH 063/182] fix(webapp): throw error status if name is absent
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 63be1429..058097e8 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -11,11 +11,11 @@ Property Name As %String [ Calculated ];
Method NameGet() As %String
{
- Set tName = $Get(..SecurityProperties("Name"), $Get(..SecurityProperties("Url"), ""))
- If tName = "" {
- Throw $$$ERROR($$$GeneralError, "Name is required for web applications")
+ Set name = $Get(..SecurityProperties("Name"), $Get(..SecurityProperties("Url"), ""))
+ If name = "" {
+ $$$ThrowStatus($$$ERROR($$$GeneralError, "Name is required for web applications"))
}
- Return tName
+ Return name
}
/// Keeps track of all application properties
From 9c39053a122835c937fcb859406cfe4d2ef22541 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:48:39 -0400
Subject: [PATCH 064/182] fix(webapp): process properties using
%EvaluateSystemExpression
---
.../IPM/ResourceProcessor/WebApplication.cls | 53 +++++--------------
src/cls/IPM/Utils/Module.cls | 15 ++++++
2 files changed, 27 insertions(+), 41 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 058097e8..40b10bd9 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -39,37 +39,6 @@ Method CopyAttributes() [ Private ]
}
}
-Method ReplaceMatchRoles(matchRoles As %String, dbDir As %String) As %String
-{
- New $NAMESPACE
- Set $NAMESPACE = "%SYS"
- Set templates = $Listbuild("{$dbrole}", "${dbrole}")
- For i=1:1:$Listlength(templates) {
- Set template = $Listget(templates, i)
- If matchRoles[template {
- Set dbRole = "%DB_DEFAULT"
- Set db = ##class(SYS.Database).%OpenId(dbDir)
- If $Isobject(db) {
- Set dbRole = db.ResourceName
- }
- Set matchRoles = $Replace(matchRoles, template, dbRole)
- }
- }
- Return matchRoles
-}
-
-Method ReplaceNameSpace(expression As %String, namespace As %String) As %String
-{
- Set templates = $Listbuild("{$ns}", "${ns}")
- Set ptr = 0
- While $LISTNEXT(templates, ptr, template) {
- If expression[template {
- Set expression = $REPLACE(expression, template, namespace)
- }
- }
- Return expression
-}
-
/// Called as phase pPhase is executed for the resource. If pResourceHandled is set to true,
/// then the default behavior for that resource will be bypassed in the current phase.
Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boolean = 0) As %Status
@@ -133,20 +102,22 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
// TODO: Make mirror-safe.
- Set tOriginalNS = $namespace
- Set dbDir = $$$defdir
New $namespace
Set $namespace = "%SYS"
-
+
Merge properties = ..SecurityProperties
- If $D(properties("MatchRoles"), tMatchRoles) # 2 {
- Set properties("MatchRoles") = ..ReplaceMatchRoles(tMatchRoles, dbDir)
+ If $Get(properties("AutheEnabled")) = "" {
+ Set properties("AutheEnabled") = (
+ ($Get(properties("PasswordAuthEnabled")) * $$$AutheCache) +
+ ($Get(properties("UnauthenticatedEnabled")) * $$$AutheUnauthenticated) +
+ ($Get(properties("DelegatedAuthEnabled")) * $$$AutheDelegated) +
+ ($Get(properties("KerberosAuthEnabled")) * $$$AutheK5API) +
+ ($Get(properties("LoginTokenEnabled")) * $$$AutheLoginToken)
+ )
}
- If $D(properties("NameSpace"), tNameSpace) {
- Set properties("NameSpace") = ..ReplaceNameSpace(tNameSpace, tOriginalNS)
- } Else {
- // Does WSGI application need a namespace?
- Set properties("NameSpace") = tOriginalNS
+
+ If properties("AutheEnabled") = 0 {
+ kill properties("AutheEnabled")
}
If ##class(Security.Applications).Exists(..Name) {
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 2c3815af..837b9c40 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2245,6 +2245,7 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$binDir} - the instance's bin directory.
/// {$libDir} - the instance's lib directory.
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
+/// {$dbrole} - the current namespace's database role
///
/// For backward compatibility, supporting both ${var} and {$var}
ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal ]
@@ -2285,6 +2286,9 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Do ##class(%Studio.General).GetWebServerPort(,,,.urlRoot)
Set result = ..%RegExReplace(result, "webroot", urlRoot)
+ Set dbRole = ..GetDatabaseRole()
+ Set result = ..%RegExReplace(result, "dbrole", dbRole)
+
Return result
}
@@ -2297,6 +2301,17 @@ ClassMethod GetDatabaseInfoForNamespace(pNamespace As %String, Output pPropertie
Set sc = $ClassMethod("Config.Namespaces", "Get", pNamespace, .pProperties)
}
+ClassMethod GetDatabaseRole()
+{
+ Set dbDir = $$$defdir
+
+ New $NAMESPACE
+ Set $NAMESPACE = "%SYS"
+
+ Set db = ##class(SYS.Database).%OpenId(dbDir)
+ Return $SELECT($IsObject(db): db.ResourceName, 1: "%DB_DEFAULT")
+}
+
ClassMethod %RegExReplace(pString As %String, pName As %String, pValue As %String) As %String [ Internal ]
{
Set tString = pString
From 0cfc1dba4182731bc559c92a6f27af208018d5cf Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:49:35 -0400
Subject: [PATCH 065/182] style: remove unused code and comment
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 40b10bd9..3bd17623 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -81,15 +81,11 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
$$$ThrowOnError(##super(pPhase,.pParams))
Set tVerbose = $Get(pParams("Verbose"))
- #; Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%ZHSLIB.PackageManager.Developer.Lifecycle.Application).%ClassName(1))
- Set tIsApplication = ..ResourceReference.Module.Lifecycle.%IsA(##class(%IPM.Lifecycle.DeployedModule).%ClassName(1))
- // TODO: Make mirror-safe?
- // Would require moving mirror-safe APIs to the package manager - at which point, why not just have the package manager manage the whole federation?
- If ((pPhase = "Configure") || ((pPhase = "Activate") && tIsApplication)) {
+ If ((pPhase = "Configure") || (pPhase = "Activate")) {
// Create web application
Do ..CreateOrUpdateWebApp(tVerbose)
- } ElseIf ((pPhase = "Unconfigure") || ((pPhase = "Clean") && tIsApplication)) {
+ } ElseIf ((pPhase = "Unconfigure") || (pPhase = "Clean")) {
// Remove web application
Do ..DeleteWebApp(tVerbose)
}
From 1eb87ee30d0d33be206d0c7a81df68830903ba2e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:50:47 -0400
Subject: [PATCH 066/182] test(webapp): modify test cases to make sure they
actually work
---
.../Test/PM/Integration/_data/rest-app/module.xml | 6 ++++--
.../_data/rest-app/src/cls/Test/Rest/Demo.cls | 13 +++++++++++++
.../Test/PM/Integration/_data/wsgi-app/module.xml | 4 +++-
3 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 6ac103a8..1a52e01c 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -4,8 +4,8 @@
rest-demo
1.0.0
- This is a demo of a flask application
- flask
+ This is a demo of a rest application
+ rest
Shuheng Liu
InterSystems
@@ -16,8 +16,10 @@
module
src
+
+
+
+}
+
+ClassMethod GetInfo() As %Status
+{
+ Write "Hello, World!"
+ Quit $$$OK
+}
+
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index 9825dd59..b9477909 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -23,10 +23,12 @@
UnauthenticatedEnabled="1"
Description="Sample WSGI application using Flask"
MatchRoles=":${dbrole}"
+ NameSpace="${ns}"
WSGIAppLocation="${libdir}flask-demo/flaskapp/"
WSGIAppName="app"
WSGICallable="app"
- />
+ DispatchClass="%SYS.Python.WSGI"
+ />
Module installed successfully!
From 56ce74e91c1c7c1b5befc3f0a12b4bede6f5bfde Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 13:51:21 -0400
Subject: [PATCH 067/182] test(webapp): add test for `get`ting REST endpoints
---
.../Test/PM/Integration/Base.cls | 24 +++++++++++++++++++
.../PM/Integration/InstallApplication.cls | 2 ++
2 files changed, 26 insertions(+)
diff --git a/tests/integration_tests/Test/PM/Integration/Base.cls b/tests/integration_tests/Test/PM/Integration/Base.cls
index 19d41c0a..57d147c5 100644
--- a/tests/integration_tests/Test/PM/Integration/Base.cls
+++ b/tests/integration_tests/Test/PM/Integration/Base.cls
@@ -125,4 +125,28 @@ ClassMethod GetModuleDir(subfolders... As %String) As %String
Quit tModuleDir
}
+ClassMethod HttpGet(pPath As %String = "/") As %Status
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ Set tSC = ##class(Config.Startup).Get(.props)
+ If $$$ISERR(tSC) {
+ Quit tSC
+ }
+
+ Set request = ##class(%Net.HttpRequest).%New()
+ Set request.Server = "127.0.0.1"
+ Set request.Port = props("WebServerPort")
+ Set path = props("WebServerURLPrefix") _ "/" _ pPath
+ Set tSC = request.Get(path)
+ If $$$ISERR(tSC) {
+ Quit tSC
+ }
+ If (request.HttpResponse.StatusCode '= 200) {
+ Quit $$$ERROR($$$GeneralError, "HTTP Status: "_request.HttpResponse.StatusCode)
+ }
+ Quit $$$OK
+}
+
}
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 8331c67c..508c1656 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -42,6 +42,7 @@ Method TestWSGIApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/wsgi-app/")
Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded WSGI Application successfully. " _ tModuleDir)
+ Do $$$AssertStatusOK(..HttpGet("my/flask/demo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
@@ -55,6 +56,7 @@ Method TestRestApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
+ Do $$$AssertStatusOK(..HttpGet("restdemo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
From cb8022c575b41b6c2ab2572faf45efe10fe908ad Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 14:01:24 -0400
Subject: [PATCH 068/182] style: fix indent using tabs
---
src/cls/IPM/Utils/Module.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 837b9c40..43d1fbcf 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2308,7 +2308,7 @@ ClassMethod GetDatabaseRole()
New $NAMESPACE
Set $NAMESPACE = "%SYS"
- Set db = ##class(SYS.Database).%OpenId(dbDir)
+ Set db = ##class(SYS.Database).%OpenId(dbDir)
Return $SELECT($IsObject(db): db.ResourceName, 1: "%DB_DEFAULT")
}
From fb787d5a80e9c634c20711e78c00bc7cfaa87110 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 14:11:57 -0400
Subject: [PATCH 069/182] debug: trigger a ci to see why HttpGet fails
---
tests/integration_tests/Test/PM/Integration/Base.cls | 2 +-
.../Test/PM/Integration/InstallApplication.cls | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/integration_tests/Test/PM/Integration/Base.cls b/tests/integration_tests/Test/PM/Integration/Base.cls
index 57d147c5..b033f0c7 100644
--- a/tests/integration_tests/Test/PM/Integration/Base.cls
+++ b/tests/integration_tests/Test/PM/Integration/Base.cls
@@ -144,7 +144,7 @@ ClassMethod HttpGet(pPath As %String = "/") As %Status
Quit tSC
}
If (request.HttpResponse.StatusCode '= 200) {
- Quit $$$ERROR($$$GeneralError, "HTTP Status: "_request.HttpResponse.StatusCode)
+ Quit $$$ERROR($$$GeneralError, "HTTP Status: "_request.HttpResponse.StatusCode_$char(13,10)_request.HttpResponse.Data.Read())
}
Quit $$$OK
}
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 508c1656..3eacb5e8 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -40,7 +40,7 @@ Method TestWSGIApp()
}
Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/wsgi-app/")
- Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
+ Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded WSGI Application successfully. " _ tModuleDir)
Do $$$AssertStatusOK(..HttpGet("my/flask/demo/"))
} Catch e {
@@ -54,7 +54,7 @@ Method TestRestApp()
Try {
Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
- Set tSC = ##class(%IPM.Main).Shell("load " _ tModuleDir)
+ Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
Do $$$AssertStatusOK(..HttpGet("restdemo/"))
} Catch e {
From 81c1e65f13076d28364093d3fec4484c6af72298 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 14:45:49 -0400
Subject: [PATCH 070/182] feat: use a better way to determine database resource
---
src/cls/IPM/Utils/Module.cls | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 43d1fbcf..027e468f 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2303,13 +2303,8 @@ ClassMethod GetDatabaseInfoForNamespace(pNamespace As %String, Output pPropertie
ClassMethod GetDatabaseRole()
{
- Set dbDir = $$$defdir
-
- New $NAMESPACE
- Set $NAMESPACE = "%SYS"
-
- Set db = ##class(SYS.Database).%OpenId(dbDir)
- Return $SELECT($IsObject(db): db.ResourceName, 1: "%DB_DEFAULT")
+ Do ##class(%SYS.Namespace).GetNSInfo($NAMESPACE, .nsInfo)
+ Return $Get(nsInfo("Resource"), "%DB_DEFAULT")
}
ClassMethod %RegExReplace(pString As %String, pName As %String, pValue As %String) As %String [ Internal ]
From 8314fc177139d57eb0b7de6f68beafd2007977ad Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 14:46:11 -0400
Subject: [PATCH 071/182] style: remove comments about mirror safety
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 3 ---
1 file changed, 3 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 3bd17623..d83bff66 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -97,7 +97,6 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
- // TODO: Make mirror-safe.
New $namespace
Set $namespace = "%SYS"
@@ -169,8 +168,6 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
/// This removes an existing CSP application
Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
- // TODO: Make mirror-safe.
-
// Only try to purge files if the directory exists.
Set sc = ##class(%Library.EnsembleMgr).deletePortal(..Name,"",pVerbose)
Set errorCodes = $System.Status.GetErrorCodes(sc)
From 590dbd2e9cf01c21b4ef27f1a58500e1042f7fda Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 9 Sep 2024 14:59:58 -0400
Subject: [PATCH 072/182] refactor: use Security.Applications API instead of
%Library.EnsembleMgr
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index d83bff66..9b392830 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -168,14 +168,11 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
/// This removes an existing CSP application
Method DeleteWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
- // Only try to purge files if the directory exists.
- Set sc = ##class(%Library.EnsembleMgr).deletePortal(..Name,"",pVerbose)
- Set errorCodes = $System.Status.GetErrorCodes(sc)
- If ((errorCodes [ $$$ApplicationDoesNotExist) || (errorCodes [ $$$DeleteObjectNotFound)) {
- // Not actually a problem - allow Clean/Uninstall to continue if it fails while trying to remove something that doesn't exist.
- Set sc = $$$OK
+ New $Namespace
+ Set $Namespace = "%SYS"
+ If ##class(Security.Applications).Exists(..Name) {
+ $$$ThrowOnError(##class(Security.Applications).Delete(..Name))
}
- $$$ThrowOnError(sc)
}
/// Returns a unique name for this resource.
From 99c46ecabd961fefef1ef9424e96040541cb88d5 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 09:51:14 -0400
Subject: [PATCH 073/182] enhance(webapp): always set IsDirectory to 1
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 9b392830..335b6b12 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -193,7 +193,7 @@ Method GetSourceControlInfo(Output pInfo As %IPM.ExtensionBase.SourceControl.Res
Set pInfo.ResourceType = "/CSP/"
Set pInfo.Prefix = ..Name
Set pInfo.RelativePath = ..ResourceReference.Name
- Set pInfo.IsDirectory = ($Piece(..ResourceReference.Name,"/",*) '[ ".")
+ Set pInfo.IsDirectory = 1
}
Quit $$$OK
}
From cf38fc0aa424d6223152bb6c0a0a886a965f8bef Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 10:05:23 -0400
Subject: [PATCH 074/182] feat(webapp): add method to create role to read
routine DB of namespace
---
.../IPM/ResourceProcessor/WebApplication.cls | 54 +++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 335b6b12..96d71b1d 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -221,4 +221,58 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
+/// Grant read permissions to the routine database. This is necessary because
+/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
+Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ // Find the namespace-default routine DB
+ $$$ThrowOnError(##class(Config.Namespaces).Get(pNamespace,.nsProps))
+ Set defaultDB = nsProps("Routines")
+ If pVerbose {
+ Write !, "Routines DB: ", defaultDB
+ }
+ $$$ThrowOnError(##class(Config.Databases).Get(defaultDB,.dbProps))
+ Set db=##Class(SYS.Database).%OpenId(dbProps("Directory"),,.sc)
+ $$$ThrowOnError(sc)
+ Set dbResource = db.ResourceName
+ If pVerbose {
+ Write !, "DB Resoure: ", dbResource
+ }
+
+ // Is there public READ permission on the namespace-default routine DB?
+ $$$ThrowOnError(##class(Security.Resources).Get(dbResource,.resourceProps))
+ If (resourceProps("PublicPermission")#2) {
+ // No need to do anything else if there's public R permission
+ If pVerbose {
+ Write !, "Public Read permission already exists on ", dbResource, !, "Skipping"
+ }
+ Quit
+ }
+
+ // Create role granting R permission on namespace-default routine DB resource
+ // This is the least-access means to be able to use the REST handler class
+ // in unauthenticated mode
+ Set roleName = $ListToString($lb("ipm", "db", dbResource, "read"), ".")
+ Set roleProps("Description") = "Role created by %IPM that grants only R permission on "_dbResource
+ Set roleProps("Resources") = dbResource_":R"
+ Set roleProps("GrantedRoles") = ""
+ If ##class(Security.Roles).Exists(roleName) {
+ If pVerbose {
+ Write !,"Updating role: ", roleName, " with properties: "
+ Zwrite roleProps
+ }
+ $$$ThrowOnError(##class(Security.Roles).Modify(roleName,.roleProps))
+ } Else {
+ If pVerbose {
+ Write !,"Creating role: ", roleName, " with properties: "
+ Zwrite roleProps
+ }
+ $$$ThrowOnError(##class(Security.Roles).Create(roleName,roleProps("Description"),roleProps("Resources"),roleProps("GrantedRoles")))
+ }
+ return roleName
+}
+
}
From c36aa586b93469dbf53fa6002cff225aec65ea62 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 10:10:13 -0400
Subject: [PATCH 075/182] test(webapp): temp. turn off testing until #564 is
fixed
---
.../Test/PM/Integration/InstallApplication.cls | 2 --
1 file changed, 2 deletions(-)
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 3eacb5e8..b9a1c82c 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -42,7 +42,6 @@ Method TestWSGIApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/wsgi-app/")
Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded WSGI Application successfully. " _ tModuleDir)
- Do $$$AssertStatusOK(..HttpGet("my/flask/demo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
@@ -56,7 +55,6 @@ Method TestRestApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
- Do $$$AssertStatusOK(..HttpGet("restdemo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
From d4c997e74f4b17db71f3abe3581ca06f4393f568 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 11:39:50 -0400
Subject: [PATCH 076/182] fix: allow specifing `AutheEnabled` using comma
separated indices
---
.../IPM/ResourceProcessor/WebApplication.cls | 18 +++++++++---------
.../PM/Integration/_data/rest-app/module.xml | 3 +--
.../PM/Integration/_data/wsgi-app/module.xml | 2 +-
3 files changed, 11 insertions(+), 12 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 335b6b12..473749d3 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -101,17 +101,17 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
Set $namespace = "%SYS"
Merge properties = ..SecurityProperties
- If $Get(properties("AutheEnabled")) = "" {
- Set properties("AutheEnabled") = (
- ($Get(properties("PasswordAuthEnabled")) * $$$AutheCache) +
- ($Get(properties("UnauthenticatedEnabled")) * $$$AutheUnauthenticated) +
- ($Get(properties("DelegatedAuthEnabled")) * $$$AutheDelegated) +
- ($Get(properties("KerberosAuthEnabled")) * $$$AutheK5API) +
- ($Get(properties("LoginTokenEnabled")) * $$$AutheLoginToken)
- )
+ If $Data(properties("AutheEnabled"), authString) # 2 {
+ Set authList = $ListFromString(authString, ",")
+ Set ptr = 0, authNumber = 0
+ While $ListNext(authList, ptr, authBit) {
+ Set authBit = $ZSTRIP(authBit, "<>W")
+ Set authNumber = authNumber + (2 ** authBit)
+ }
+ Set properties("AutheEnabled") = authNumber
}
- If properties("AutheEnabled") = 0 {
+ If +$Get(properties("AutheEnabled")) = 0 {
kill properties("AutheEnabled")
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 1a52e01c..ac61a22f 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -24,8 +24,7 @@
Recurse="1"
Directory="{$cspdir}/restdemo"
MatchRoles=":{$dbrole}"
- PasswordAuthEnabled="1"
- UnauthenticatedEnabled="1"
+ AutheEnabled="6,5"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
CookiePath="/restdemo"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index b9477909..3212f1d5 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -20,7 +20,7 @@
Date: Thu, 12 Sep 2024 11:40:09 -0400
Subject: [PATCH 077/182] test(webapp): add more match roles for tests
---
.../Test/PM/Integration/_data/rest-app/module.xml | 2 +-
.../Test/PM/Integration/_data/wsgi-app/module.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index ac61a22f..a7f105c4 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":{$dbrole}"
+ MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
AutheEnabled="6,5"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index 3212f1d5..f0922103 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -22,7 +22,7 @@
Url="/my/flask/demo"
AutheEnabled="6,5"
Description="Sample WSGI application using Flask"
- MatchRoles=":${dbrole}"
+ MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
NameSpace="${ns}"
WSGIAppLocation="${libdir}flask-demo/flaskapp/"
WSGIAppName="app"
From 1e22663efacfd1facfdcc543621ed4dad2d6cc76 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 11:47:13 -0400
Subject: [PATCH 078/182] chore: update changelog to include generic webapp
feature
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c111b5fb..ea13587b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
+- #562 Added a generic resource processpor `WebApplication`, which handles creating and removal of all Security.Applications resources
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
From 0bd918369b5053f3658ed7201e6668722d312d59 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 14:13:41 -0400
Subject: [PATCH 079/182] feat(webapp): create and delete role for code read
---
.../IPM/ResourceProcessor/WebApplication.cls | 77 ++++++++++++++++---
.../PM/Integration/_data/rest-app/module.xml | 2 +-
2 files changed, 66 insertions(+), 13 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 2df8cc2f..84221d04 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -88,6 +88,7 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
} ElseIf ((pPhase = "Unconfigure") || (pPhase = "Clean")) {
// Remove web application
Do ..DeleteWebApp(tVerbose)
+ Do ..DeleteCodeReadRole($Get(..SecurityProperties("NameSpace")), tVerbose)
}
} Catch e {
Set sc = e.AsStatus()
@@ -115,6 +116,11 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
kill properties("AutheEnabled")
}
+ // Replace ${codeReadRole} with a role (potentially to be created) with read permission to routine database
+ If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
+ Set properties("MatchRoles") = ..ReplaceCodeReadRole(mr, ns, pVerbose)
+ }
+
If ##class(Security.Applications).Exists(..Name) {
Write:pVerbose !,"Updating Web Application ",..Name
Set sc = ##class(Security.Applications).Get(..Name,.oldProperties)
@@ -221,9 +227,25 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
-/// Grant read permissions to the routine database. This is necessary because
-/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
-Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+ClassMethod ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
+{
+ Return $ListToString($lb("IPM", "readonly", pDbResource), ".")
+}
+
+ClassMethod ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+{
+ Set templateLists = $Listbuild("{$codeReadRole}", "${codeReadRole}")
+ Set role = "", ptr = 0 // We don't create the role unless ${codeReadRole} is explicitly in the MatchRoles
+ While $ListNext(templateLists, ptr, template) {
+ If pMatchRoles [ template {
+ Set role = $Select(role="": ..CreateCodeReadRole(pNamespace, pVerbose), 1: role)
+ Set pMatchRoles = $Replace(pMatchRoles, template, role)
+ }
+ }
+ Return pMatchRoles
+}
+
+ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private ]
{
New $Namespace
Set $Namespace = "%SYS"
@@ -231,13 +253,20 @@ Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Interna
// Find the namespace-default routine DB
$$$ThrowOnError(##class(Config.Namespaces).Get(pNamespace,.nsProps))
Set defaultDB = nsProps("Routines")
- If pVerbose {
- Write !, "Routines DB: ", defaultDB
- }
$$$ThrowOnError(##class(Config.Databases).Get(defaultDB,.dbProps))
Set db=##Class(SYS.Database).%OpenId(dbProps("Directory"),,.sc)
$$$ThrowOnError(sc)
- Set dbResource = db.ResourceName
+ Return db.ResourceName
+}
+
+/// Grant read permissions to the routine database. This is necessary because
+/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
+ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ set dbResource = ..GetDbResource(pNamespace)
If pVerbose {
Write !, "DB Resoure: ", dbResource
}
@@ -249,30 +278,54 @@ Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Interna
If pVerbose {
Write !, "Public Read permission already exists on ", dbResource, !, "Skipping"
}
- Quit
+ Return ""
}
// Create role granting R permission on namespace-default routine DB resource
// This is the least-access means to be able to use the REST handler class
// in unauthenticated mode
- Set roleName = $ListToString($lb("ipm", "db", dbResource, "read"), ".")
+ Set roleName = ..ConstructRoleName(dbResource)
Set roleProps("Description") = "Role created by %IPM that grants only R permission on "_dbResource
Set roleProps("Resources") = dbResource_":R"
Set roleProps("GrantedRoles") = ""
If ##class(Security.Roles).Exists(roleName) {
If pVerbose {
- Write !,"Updating role: ", roleName, " with properties: "
+ Write !,"Updating role: ", roleName, " with properties: ", !
Zwrite roleProps
}
$$$ThrowOnError(##class(Security.Roles).Modify(roleName,.roleProps))
} Else {
If pVerbose {
- Write !,"Creating role: ", roleName, " with properties: "
+ Write !,"Creating role: ", roleName, " with properties: ", !
Zwrite roleProps
}
$$$ThrowOnError(##class(Security.Roles).Create(roleName,roleProps("Description"),roleProps("Resources"),roleProps("GrantedRoles")))
}
- return roleName
+ Return roleName
+}
+
+ClassMethod DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ Set dbResource = ..GetDbResource(pNamespace)
+ If pVerbose {
+ Write !, "DB Resoure: ", dbResource
+ }
+ Set roleName = ..ConstructRoleName(dbResource)
+ If pVerbose {
+ Write !, "Role to delete: ", roleName
+ }
+
+ If ##class(Security.Roles).Exists(roleName) {
+ $$$ThrowOnError(##class(Security.Roles).Delete(roleName))
+ If pVerbose {
+ Write !, "Deleted succesfully"
+ }
+ } ElseIf pVerbose {
+ Write !, "Skipped because role does not exist"
+ }
}
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index a7f105c4..36818524 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeReadRole}:%Developer,%All:%Manager"
AutheEnabled="6,5"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
From 1c3892153759780fde94b62b64724e6a327a3a51 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 14:52:27 -0400
Subject: [PATCH 080/182] feat(webapp): allow more flexible expression in
AutheEnabled
---
.../IPM/ResourceProcessor/WebApplication.cls | 29 ++++++++++++-------
src/cls/IPM/Utils/Module.cls | 9 ++++++
.../PM/Integration/_data/rest-app/module.xml | 2 +-
.../PM/Integration/_data/wsgi-app/module.xml | 2 +-
4 files changed, 30 insertions(+), 12 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 473749d3..62312134 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -95,22 +95,31 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Quit sc
}
+Method ComputeAutheNumber(pAuth As %String) As %String
+{
+ Set pAuth = $ZSTRIP(pAuth, "<>W")
+ If ($Extract(pAuth, 1, 2) = "#{") && ($Extract(pAuth, *) = "}") {
+ Set output = 0
+ Set pAuth = $Extract(pAuth, 3, *-1)
+ Set pAuth = $ListFromString(pAuth, "+")
+ Set ptr = 0
+ While $LISTNEXT(pAuth, ptr, component) {
+ Set component = $ZSTRIP(component, "<>W")
+ Set output = output + component
+ }
+ Return output
+ } Else {
+ Return pAuth
+ }
+}
+
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
New $namespace
Set $namespace = "%SYS"
Merge properties = ..SecurityProperties
- If $Data(properties("AutheEnabled"), authString) # 2 {
- Set authList = $ListFromString(authString, ",")
- Set ptr = 0, authNumber = 0
- While $ListNext(authList, ptr, authBit) {
- Set authBit = $ZSTRIP(authBit, "<>W")
- Set authNumber = authNumber + (2 ** authBit)
- }
- Set properties("AutheEnabled") = authNumber
- }
-
+ Set properties("AutheEnabled") = ..ComputeAutheNumber($Get(properties("AutheEnabled")))
If +$Get(properties("AutheEnabled")) = 0 {
kill properties("AutheEnabled")
}
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 027e468f..f163ad7d 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2289,6 +2289,15 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set dbRole = ..GetDatabaseRole()
Set result = ..%RegExReplace(result, "dbrole", dbRole)
+ Set result = ..%RegExReplace(result, "authK5API", $$$AutheK5API)
+ Set result = ..%RegExReplace(result, "authPassword", $$$AuthePassword)
+ Set result = ..%RegExReplace(result, "authUnauthenticated", $$$AutheUnauthenticated)
+ Set result = ..%RegExReplace(result, "authLDAP", $$$AutheLDAP)
+ Set result = ..%RegExReplace(result, "authDelegated", $$$AutheDelegated)
+ Set result = ..%RegExReplace(result, "authLoginToken", $$$AutheLoginToken)
+ Set result = ..%RegExReplace(result, "authTwoFactorSMS", $$$AutheTwoFactorSMS)
+ Set result = ..%RegExReplace(result, "authTwoFactorPW", $$$AutheTwoFactorPW)
+
Return result
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index a7f105c4..8f58dc10 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -24,7 +24,7 @@
Recurse="1"
Directory="{$cspdir}/restdemo"
MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
- AutheEnabled="6,5"
+ AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
CookiePath="/restdemo"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index f0922103..e45b4d12 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -20,7 +20,7 @@
Date: Thu, 12 Sep 2024 15:20:25 -0400
Subject: [PATCH 081/182] fix: use AuthCache for compatibility with older IRIS
---
src/cls/IPM/Utils/Module.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index f163ad7d..ee343724 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2290,7 +2290,7 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set result = ..%RegExReplace(result, "dbrole", dbRole)
Set result = ..%RegExReplace(result, "authK5API", $$$AutheK5API)
- Set result = ..%RegExReplace(result, "authPassword", $$$AuthePassword)
+ Set result = ..%RegExReplace(result, "authPassword", $$$AutheCache)
Set result = ..%RegExReplace(result, "authUnauthenticated", $$$AutheUnauthenticated)
Set result = ..%RegExReplace(result, "authLDAP", $$$AutheLDAP)
Set result = ..%RegExReplace(result, "authDelegated", $$$AutheDelegated)
From d22f12f22b6560b1bcd0c18c688c771f34ca933a Mon Sep 17 00:00:00 2001
From: "Shuheng Liu (InterSystems)"
<155992272+isc-shuliu@users.noreply.github.com>
Date: Thu, 12 Sep 2024 15:44:12 -0400
Subject: [PATCH 082/182] Revert "V1 feat webapp routine read role"
---
.../IPM/ResourceProcessor/WebApplication.cls | 107 ------------------
.../PM/Integration/_data/rest-app/module.xml | 2 +-
2 files changed, 1 insertion(+), 108 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 3564a90e..62312134 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -88,7 +88,6 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
} ElseIf ((pPhase = "Unconfigure") || (pPhase = "Clean")) {
// Remove web application
Do ..DeleteWebApp(tVerbose)
- Do ..DeleteCodeReadRole($Get(..SecurityProperties("NameSpace")), tVerbose)
}
} Catch e {
Set sc = e.AsStatus()
@@ -125,11 +124,6 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
kill properties("AutheEnabled")
}
- // Replace ${codeReadRole} with a role (potentially to be created) with read permission to routine database
- If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
- Set properties("MatchRoles") = ..ReplaceCodeReadRole(mr, ns, pVerbose)
- }
-
If ##class(Security.Applications).Exists(..Name) {
Write:pVerbose !,"Updating Web Application ",..Name
Set sc = ##class(Security.Applications).Get(..Name,.oldProperties)
@@ -236,105 +230,4 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
-ClassMethod ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
-{
- Return $ListToString($lb("IPM", "readonly", pDbResource), ".")
-}
-
-ClassMethod ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
-{
- Set templateLists = $Listbuild("{$codeReadRole}", "${codeReadRole}")
- Set role = "", ptr = 0 // We don't create the role unless ${codeReadRole} is explicitly in the MatchRoles
- While $ListNext(templateLists, ptr, template) {
- If pMatchRoles [ template {
- Set role = $Select(role="": ..CreateCodeReadRole(pNamespace, pVerbose), 1: role)
- Set pMatchRoles = $Replace(pMatchRoles, template, role)
- }
- }
- Return pMatchRoles
-}
-
-ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- // Find the namespace-default routine DB
- $$$ThrowOnError(##class(Config.Namespaces).Get(pNamespace,.nsProps))
- Set defaultDB = nsProps("Routines")
- $$$ThrowOnError(##class(Config.Databases).Get(defaultDB,.dbProps))
- Set db=##Class(SYS.Database).%OpenId(dbProps("Directory"),,.sc)
- $$$ThrowOnError(sc)
- Return db.ResourceName
-}
-
-/// Grant read permissions to the routine database. This is necessary because
-/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
-ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- set dbResource = ..GetDbResource(pNamespace)
- If pVerbose {
- Write !, "DB Resoure: ", dbResource
- }
-
- // Is there public READ permission on the namespace-default routine DB?
- $$$ThrowOnError(##class(Security.Resources).Get(dbResource,.resourceProps))
- If (resourceProps("PublicPermission")#2) {
- // No need to do anything else if there's public R permission
- If pVerbose {
- Write !, "Public Read permission already exists on ", dbResource, !, "Skipping"
- }
- Return ""
- }
-
- // Create role granting R permission on namespace-default routine DB resource
- // This is the least-access means to be able to use the REST handler class
- // in unauthenticated mode
- Set roleName = ..ConstructRoleName(dbResource)
- Set roleProps("Description") = "Role created by %IPM that grants only R permission on "_dbResource
- Set roleProps("Resources") = dbResource_":R"
- Set roleProps("GrantedRoles") = ""
- If ##class(Security.Roles).Exists(roleName) {
- If pVerbose {
- Write !,"Updating role: ", roleName, " with properties: ", !
- Zwrite roleProps
- }
- $$$ThrowOnError(##class(Security.Roles).Modify(roleName,.roleProps))
- } Else {
- If pVerbose {
- Write !,"Creating role: ", roleName, " with properties: ", !
- Zwrite roleProps
- }
- $$$ThrowOnError(##class(Security.Roles).Create(roleName,roleProps("Description"),roleProps("Resources"),roleProps("GrantedRoles")))
- }
- Return roleName
-}
-
-ClassMethod DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- Set dbResource = ..GetDbResource(pNamespace)
- If pVerbose {
- Write !, "DB Resoure: ", dbResource
- }
- Set roleName = ..ConstructRoleName(dbResource)
- If pVerbose {
- Write !, "Role to delete: ", roleName
- }
-
- If ##class(Security.Roles).Exists(roleName) {
- $$$ThrowOnError(##class(Security.Roles).Delete(roleName))
- If pVerbose {
- Write !, "Deleted succesfully"
- }
- } ElseIf pVerbose {
- Write !, "Skipped because role does not exist"
- }
-}
-
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 03a8fc15..8f58dc10 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:${codeReadRole}:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
From 87166c7763cb9ce74d2983662bb5b418504d1d4c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 15:58:43 -0400
Subject: [PATCH 083/182] feat:l reimplement codeReadRole
---
.../IPM/ResourceProcessor/WebApplication.cls | 107 ++++++++++++++++++
.../PM/Integration/_data/rest-app/module.xml | 2 +-
2 files changed, 108 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 62312134..3564a90e 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -88,6 +88,7 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
} ElseIf ((pPhase = "Unconfigure") || (pPhase = "Clean")) {
// Remove web application
Do ..DeleteWebApp(tVerbose)
+ Do ..DeleteCodeReadRole($Get(..SecurityProperties("NameSpace")), tVerbose)
}
} Catch e {
Set sc = e.AsStatus()
@@ -124,6 +125,11 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
kill properties("AutheEnabled")
}
+ // Replace ${codeReadRole} with a role (potentially to be created) with read permission to routine database
+ If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
+ Set properties("MatchRoles") = ..ReplaceCodeReadRole(mr, ns, pVerbose)
+ }
+
If ##class(Security.Applications).Exists(..Name) {
Write:pVerbose !,"Updating Web Application ",..Name
Set sc = ##class(Security.Applications).Get(..Name,.oldProperties)
@@ -230,4 +236,105 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
+ClassMethod ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
+{
+ Return $ListToString($lb("IPM", "readonly", pDbResource), ".")
+}
+
+ClassMethod ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+{
+ Set templateLists = $Listbuild("{$codeReadRole}", "${codeReadRole}")
+ Set role = "", ptr = 0 // We don't create the role unless ${codeReadRole} is explicitly in the MatchRoles
+ While $ListNext(templateLists, ptr, template) {
+ If pMatchRoles [ template {
+ Set role = $Select(role="": ..CreateCodeReadRole(pNamespace, pVerbose), 1: role)
+ Set pMatchRoles = $Replace(pMatchRoles, template, role)
+ }
+ }
+ Return pMatchRoles
+}
+
+ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ // Find the namespace-default routine DB
+ $$$ThrowOnError(##class(Config.Namespaces).Get(pNamespace,.nsProps))
+ Set defaultDB = nsProps("Routines")
+ $$$ThrowOnError(##class(Config.Databases).Get(defaultDB,.dbProps))
+ Set db=##Class(SYS.Database).%OpenId(dbProps("Directory"),,.sc)
+ $$$ThrowOnError(sc)
+ Return db.ResourceName
+}
+
+/// Grant read permissions to the routine database. This is necessary because
+/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
+ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ set dbResource = ..GetDbResource(pNamespace)
+ If pVerbose {
+ Write !, "DB Resoure: ", dbResource
+ }
+
+ // Is there public READ permission on the namespace-default routine DB?
+ $$$ThrowOnError(##class(Security.Resources).Get(dbResource,.resourceProps))
+ If (resourceProps("PublicPermission")#2) {
+ // No need to do anything else if there's public R permission
+ If pVerbose {
+ Write !, "Public Read permission already exists on ", dbResource, !, "Skipping"
+ }
+ Return ""
+ }
+
+ // Create role granting R permission on namespace-default routine DB resource
+ // This is the least-access means to be able to use the REST handler class
+ // in unauthenticated mode
+ Set roleName = ..ConstructRoleName(dbResource)
+ Set roleProps("Description") = "Role created by %IPM that grants only R permission on "_dbResource
+ Set roleProps("Resources") = dbResource_":R"
+ Set roleProps("GrantedRoles") = ""
+ If ##class(Security.Roles).Exists(roleName) {
+ If pVerbose {
+ Write !,"Updating role: ", roleName, " with properties: ", !
+ Zwrite roleProps
+ }
+ $$$ThrowOnError(##class(Security.Roles).Modify(roleName,.roleProps))
+ } Else {
+ If pVerbose {
+ Write !,"Creating role: ", roleName, " with properties: ", !
+ Zwrite roleProps
+ }
+ $$$ThrowOnError(##class(Security.Roles).Create(roleName,roleProps("Description"),roleProps("Resources"),roleProps("GrantedRoles")))
+ }
+ Return roleName
+}
+
+ClassMethod DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+{
+ New $Namespace
+ Set $Namespace = "%SYS"
+
+ Set dbResource = ..GetDbResource(pNamespace)
+ If pVerbose {
+ Write !, "DB Resoure: ", dbResource
+ }
+ Set roleName = ..ConstructRoleName(dbResource)
+ If pVerbose {
+ Write !, "Role to delete: ", roleName
+ }
+
+ If ##class(Security.Roles).Exists(roleName) {
+ $$$ThrowOnError(##class(Security.Roles).Delete(roleName))
+ If pVerbose {
+ Write !, "Deleted succesfully"
+ }
+ } ElseIf pVerbose {
+ Write !, "Skipped because role does not exist"
+ }
+}
+
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 8f58dc10..03a8fc15 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeReadRole}:%Developer,%All:%Manager"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
From ccb9be8bc20f53b1e22a54cc037b23d6fb82b697 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 12 Sep 2024 15:59:40 -0400
Subject: [PATCH 084/182] test: re-add test case for HTTP GET
---
.../Test/PM/Integration/InstallApplication.cls | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index b9a1c82c..3eacb5e8 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -42,6 +42,7 @@ Method TestWSGIApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/wsgi-app/")
Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded WSGI Application successfully. " _ tModuleDir)
+ Do $$$AssertStatusOK(..HttpGet("my/flask/demo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
@@ -55,6 +56,7 @@ Method TestRestApp()
set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
+ Do $$$AssertStatusOK(..HttpGet("restdemo/"))
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
From bcf2167aa4a8d9c42738a2450216718120730557 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 13 Sep 2024 14:07:00 -0400
Subject: [PATCH 085/182] style: typo fixes
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 3564a90e..ad95e073 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -277,7 +277,7 @@ ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %
set dbResource = ..GetDbResource(pNamespace)
If pVerbose {
- Write !, "DB Resoure: ", dbResource
+ Write !, "DB Resource: ", dbResource
}
// Is there public READ permission on the namespace-default routine DB?
@@ -294,7 +294,7 @@ ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %
// This is the least-access means to be able to use the REST handler class
// in unauthenticated mode
Set roleName = ..ConstructRoleName(dbResource)
- Set roleProps("Description") = "Role created by %IPM that grants only R permission on "_dbResource
+ Set roleProps("Description") = "Role created by IPM that grants only R permission on "_dbResource
Set roleProps("Resources") = dbResource_":R"
Set roleProps("GrantedRoles") = ""
If ##class(Security.Roles).Exists(roleName) {
@@ -320,7 +320,7 @@ ClassMethod DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ In
Set dbResource = ..GetDbResource(pNamespace)
If pVerbose {
- Write !, "DB Resoure: ", dbResource
+ Write !, "DB Resource: ", dbResource
}
Set roleName = ..ConstructRoleName(dbResource)
If pVerbose {
From 59482c68f1df84fa9a71a8051306dc00352addc6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Sep 2024 09:47:40 -0400
Subject: [PATCH 086/182] enhance(webapp): include module name in
${codeReadRole}, so it's unique
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index ad95e073..042067c9 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -236,12 +236,14 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
-ClassMethod ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
+Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
{
- Return $ListToString($lb("IPM", "readonly", pDbResource), ".")
+ // Include the module name in the role name to avoid conflicts
+ // This role should only be managed by IPM itself
+ Return $ListToString($lb("IPM", "readonly", ..ResourceReference.Module.Name, pDbResource), ".")
}
-ClassMethod ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
{
Set templateLists = $Listbuild("{$codeReadRole}", "${codeReadRole}")
Set role = "", ptr = 0 // We don't create the role unless ${codeReadRole} is explicitly in the MatchRoles
@@ -270,7 +272,7 @@ ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private
/// Grant read permissions to the routine database. This is necessary because
/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
-ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
+Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
{
New $Namespace
Set $Namespace = "%SYS"
@@ -294,7 +296,7 @@ ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %
// This is the least-access means to be able to use the REST handler class
// in unauthenticated mode
Set roleName = ..ConstructRoleName(dbResource)
- Set roleProps("Description") = "Role created by IPM that grants only R permission on "_dbResource
+ Set roleProps("Description") = "Role created by IPM that grants only R permission on "_dbResource _ ". This role should only be managed by IPM."
Set roleProps("Resources") = dbResource_":R"
Set roleProps("GrantedRoles") = ""
If ##class(Security.Roles).Exists(roleName) {
@@ -313,7 +315,7 @@ ClassMethod CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %
Return roleName
}
-ClassMethod DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+Method DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
{
New $Namespace
Set $Namespace = "%SYS"
From d109d7c6d561fa0b48875c5db8676629025414a6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Sep 2024 09:48:20 -0400
Subject: [PATCH 087/182] test(webapp): add test case for 2 webapps sharing the
same namespace
---
.../PM/Integration/InstallApplication.cls | 24 +++++++++++--
.../PM/Integration/_data/rest-app2/module.xml | 36 +++++++++++++++++++
.../rest-app2/src/cls/Test/Rest/Demo2.cls | 17 +++++++++
3 files changed, 74 insertions(+), 3 deletions(-)
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 3eacb5e8..35efd361 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -53,10 +53,28 @@ Method TestRestApp()
Set tSC = $$$OK
Try {
Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
- set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
- Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir)
- Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir)
+
+ Set tModuleDir1 = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app/")
+ Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir1)
+ Do $$$AssertStatusOK(tSC,"Loaded REST Application ""restdemo/"" successfully. " _ tModuleDir1)
+ Do $$$AssertStatusOK(..HttpGet("restdemo/"))
+
+ // Install another REST application of the same database
+ // Uninstalling this module shouldn't affect the first one (beacuse ${codeReadRole} is unique for each module)
+ Set tModuleDir2 = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app2/")
+ Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir2)
+ Do $$$AssertStatusOK(tSC,"Loaded REST Application ""restdemo2/"" successfully. " _ tModuleDir2)
+ Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir2)
+ Do $$$AssertStatusOK(..HttpGet("restdemo2/"))
+
+ // Delete the 2nd REST application, the 1st one should still be accessible
+ Set tSC = ##class(%IPM.Main).Shell("uninstall -verbose rest-demo2")
+ Do $$$AssertStatusOK(tSC,"Deleted REST Application ""restdemo2/"" successfully. " _ tModuleDir2)
Do $$$AssertStatusOK(..HttpGet("restdemo/"))
+
+ // Delete the 1st REST application
+ Set tSC = ##class(%IPM.Main).Shell("uninstall -verbose rest-demo")
+ Do $$$AssertStatusOK(tSC,"Deleted REST Application ""restdemo/"" successfully. " _ tModuleDir1)
} Catch e {
Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
new file mode 100644
index 00000000..d6f092b8
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
@@ -0,0 +1,36 @@
+
+
+
+
+ rest-demo2
+ 1.0.0
+ This is a demo of a rest application
+ rest
+
+ Shuheng Liu
+ InterSystems
+ 2024
+ MIT
+ notes
+
+ module
+ src
+
+
+
+ Module installed successfully!
+
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
new file mode 100644
index 00000000..8aa974a2
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
@@ -0,0 +1,17 @@
+Class Test.Rest.Demo2 Extends %CSP.REST
+{
+
+XData UrlMap [ XMLNamespace = "http://www.intersystems.com/urlmap" ]
+{
+
+
+
+}
+
+ClassMethod GetInfo() As %Status
+{
+ Write "Hello, World!"
+ Quit $$$OK
+}
+
+}
From e0da977cd9f1dad64544e7d9045eb264996b042a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Sep 2024 10:38:34 -0400
Subject: [PATCH 088/182] fix: fix %Evaluate() so COS expression evaluates
properly
---
src/cls/IPM/Storage/Module.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls
index d7c14613..73a0a060 100644
--- a/src/cls/IPM/Storage/Module.cls
+++ b/src/cls/IPM/Storage/Module.cls
@@ -1434,7 +1434,7 @@ Method %Evaluate(pAttrValue As %String, ByRef pParams) As %String [ Internal ]
Set name = ..Name
Set attrValue = ##class(%Regex.Matcher).%New("(?i)\{name\}", attrValue).ReplaceAll($Replace(name,"\","\\"))
- Set regex = ##class(%Regex.Matcher).%New("#\{([^}]+)\}", tAttrValue)
+ Set regex = ##class(%Regex.Matcher).%New("#\{([^}]+)\}", attrValue)
While regex.Locate() {
Set expr = regex.Group(1)
Set value = ..%EvaluateExpression(expr)
From bced536f8df54bb9882a1e07314f62f4e5b70196 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Sep 2024 10:40:12 -0400
Subject: [PATCH 089/182] fix: strip leading space in #{expr} so `return @expr`
is always valid
---
src/cls/IPM/Storage/Module.cls | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls
index 73a0a060..2992aca7 100644
--- a/src/cls/IPM/Storage/Module.cls
+++ b/src/cls/IPM/Storage/Module.cls
@@ -1448,6 +1448,8 @@ Method %Evaluate(pAttrValue As %String, ByRef pParams) As %String [ Internal ]
Method %EvaluateExpression(pExpr) As %String [ Internal ]
{
Try {
+ // Remove leading whitespace because there can only be 1 whitespace between "return" and its argument
+ set pExpr = $Zstrip(pExpr, "
Date: Tue, 17 Sep 2024 10:42:10 -0400
Subject: [PATCH 090/182] feat(webapp): handle `#{expr}` in abstract processor
%Evaluate()
---
src/cls/IPM/ResourceProcessor/Abstract.cls | 9 ++++++++
.../IPM/ResourceProcessor/WebApplication.cls | 22 -------------------
.../PM/Integration/_data/wsgi-app/module.xml | 2 +-
3 files changed, 10 insertions(+), 23 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls
index 5a3d0ed2..9b596b8d 100644
--- a/src/cls/IPM/ResourceProcessor/Abstract.cls
+++ b/src/cls/IPM/ResourceProcessor/Abstract.cls
@@ -261,6 +261,15 @@ Method %Evaluate(pAttrValue As %String) As %String [ Internal ]
// {name} expression does not contain dollar sign; handle separately
Set name = ..ResourceReference.Name
Set attrValue = ##class(%Regex.Matcher).%New("(?i)\{name\}", attrValue).ReplaceAll($Replace(name,"\","\\"))
+
+ Set regex = ##class(%Regex.Matcher).%New("#\{([^}]+)\}", attrValue)
+ While regex.Locate() {
+ Set expr = regex.Group(1)
+ Set value = ..ResourceReference.Module.%EvaluateExpression(expr)
+ Set $Extract(attrValue, regex.Start, regex.End - 1) = value
+ Set regex.Text = attrValue
+ }
+
Return attrValue
}
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 042067c9..204a07b0 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -96,34 +96,12 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Quit sc
}
-Method ComputeAutheNumber(pAuth As %String) As %String
-{
- Set pAuth = $ZSTRIP(pAuth, "<>W")
- If ($Extract(pAuth, 1, 2) = "#{") && ($Extract(pAuth, *) = "}") {
- Set output = 0
- Set pAuth = $Extract(pAuth, 3, *-1)
- Set pAuth = $ListFromString(pAuth, "+")
- Set ptr = 0
- While $LISTNEXT(pAuth, ptr, component) {
- Set component = $ZSTRIP(component, "<>W")
- Set output = output + component
- }
- Return output
- } Else {
- Return pAuth
- }
-}
-
Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
{
New $namespace
Set $namespace = "%SYS"
Merge properties = ..SecurityProperties
- Set properties("AutheEnabled") = ..ComputeAutheNumber($Get(properties("AutheEnabled")))
- If +$Get(properties("AutheEnabled")) = 0 {
- kill properties("AutheEnabled")
- }
// Replace ${codeReadRole} with a role (potentially to be created) with read permission to routine database
If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index e45b4d12..668e160e 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -20,7 +20,7 @@
Date: Wed, 18 Sep 2024 14:57:55 -0400
Subject: [PATCH 091/182] docs: improve documentation for templates in
%Evaluate
---
src/cls/IPM/Utils/Module.cls | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index ee343724..d318b071 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2245,7 +2245,15 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$binDir} - the instance's bin directory.
/// {$libDir} - the instance's lib directory.
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
-/// {$dbrole} - the current namespace's database role
+/// {$dbrole} - the current namespace's globals database role
+/// {$codeDbRole} - the current namespace's routine database role (will be created if needed)
+/// {$authK5API} - the authentication macro $$$AutheK5API used for Security.Applications. Its value is equal to 4 (2^2).
+/// {$authPassword} - the authentication macro $$$AutheCache used for Security.Applications. Its value is equal to 32 (2^5).
+/// {$authLDAP} - the authentication macro $$$AutheLDAP used for Security.Applications. Its value is equal to 2048 (2^11).
+/// {$authDelegated} - the authentication macro $$$AutheDelegated used for Security.Applications. Its value is equal to 8192 (2^13).
+/// {$authLoginToken} - the authentication macro $$$AutheLogintToken used for Security.Applications. Its value is equal to 16384 (2^14).
+/// {$authTwoFactorSMS} - the authentication macro $$$AutheTwoFactorSMS used for Security.Applications. Its value is equal to 1048576 (2^20).
+/// {$authTwoFactorPW} - the authentication macro $$$AutheTwoFactorPW used for Security.Applications. Its value is equal to 2097152 (2^21).
///
/// For backward compatibility, supporting both ${var} and {$var}
ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal ]
@@ -2289,6 +2297,7 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set dbRole = ..GetDatabaseRole()
Set result = ..%RegExReplace(result, "dbrole", dbRole)
+ // Authentication bits used by Security.Applications
Set result = ..%RegExReplace(result, "authK5API", $$$AutheK5API)
Set result = ..%RegExReplace(result, "authPassword", $$$AutheCache)
Set result = ..%RegExReplace(result, "authUnauthenticated", $$$AutheUnauthenticated)
From e9dcfd4435410df50f142f5d2e752b4cde9fb0e4 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 15:20:06 -0400
Subject: [PATCH 092/182] feat: rename codeDbReadRole and globalsDbRole in
templates
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 6 +++---
src/cls/IPM/Utils/Module.cls | 5 +++--
.../Test/PM/Integration/InstallApplication.cls | 2 +-
.../Test/PM/Integration/_data/rest-app/module.xml | 2 +-
.../Test/PM/Integration/_data/rest-app2/module.xml | 2 +-
5 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 204a07b0..5cb885bd 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -103,7 +103,7 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
Merge properties = ..SecurityProperties
- // Replace ${codeReadRole} with a role (potentially to be created) with read permission to routine database
+ // Replace ${codeDbReadRole} with a role (potentially to be created) with read permission to routine database
If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
Set properties("MatchRoles") = ..ReplaceCodeReadRole(mr, ns, pVerbose)
}
@@ -223,8 +223,8 @@ Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private
Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
{
- Set templateLists = $Listbuild("{$codeReadRole}", "${codeReadRole}")
- Set role = "", ptr = 0 // We don't create the role unless ${codeReadRole} is explicitly in the MatchRoles
+ Set templateLists = $Listbuild("{$codeDbReadRole}", "${codeDbReadRole}")
+ Set role = "", ptr = 0 // We don't create the role unless ${codeDbReadRole} is explicitly in the MatchRoles
While $ListNext(templateLists, ptr, template) {
If pMatchRoles [ template {
Set role = $Select(role="": ..CreateCodeReadRole(pNamespace, pVerbose), 1: role)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index d318b071..985cfcb6 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2245,8 +2245,9 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$binDir} - the instance's bin directory.
/// {$libDir} - the instance's lib directory.
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
-/// {$dbrole} - the current namespace's globals database role
-/// {$codeDbRole} - the current namespace's routine database role (will be created if needed)
+/// {$dbrole} - (deprecated in favor of {$globalsDbRole}) the current namespace's globals database role
+/// {$globalsDbRole} - the current namespace's globals database role, functionally equivalent to {$dbrole} in legacy packages.
+/// {$codeDbReadRole} - the current namespace's routine database role (will be created if needed)
/// {$authK5API} - the authentication macro $$$AutheK5API used for Security.Applications. Its value is equal to 4 (2^2).
/// {$authPassword} - the authentication macro $$$AutheCache used for Security.Applications. Its value is equal to 32 (2^5).
/// {$authLDAP} - the authentication macro $$$AutheLDAP used for Security.Applications. Its value is equal to 2048 (2^11).
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 35efd361..7dce3885 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -60,7 +60,7 @@ Method TestRestApp()
Do $$$AssertStatusOK(..HttpGet("restdemo/"))
// Install another REST application of the same database
- // Uninstalling this module shouldn't affect the first one (beacuse ${codeReadRole} is unique for each module)
+ // Uninstalling this module shouldn't affect the first one (beacuse ${codeDbReadRole} is unique for each module)
Set tModuleDir2 = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app2/")
Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir2)
Do $$$AssertStatusOK(tSC,"Loaded REST Application ""restdemo2/"" successfully. " _ tModuleDir2)
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 03a8fc15..878c09b3 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:${codeReadRole}:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%All:%Manager"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
index d6f092b8..7ef87f09 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo2"
- MatchRoles=":${dbrole}:%SQL:${codeReadRole}:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%All:%Manager"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo2"
ServeFiles="1"
From 2a58e3b3ccd6d70f2a240582238e328f1c55283a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 15:22:49 -0400
Subject: [PATCH 093/182] test(webapp): improve web app match roles so they
make more sense
---
.../Test/PM/Integration/_data/rest-app/module.xml | 2 +-
.../Test/PM/Integration/_data/rest-app2/module.xml | 2 +-
.../Test/PM/Integration/_data/wsgi-app/module.xml | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 878c09b3..7c21806a 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%Manager:%All"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
index 7ef87f09..4a76f620 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo2"
- MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%Manager:%All"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo2"
ServeFiles="1"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
index 668e160e..92032894 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/wsgi-app/module.xml
@@ -22,7 +22,7 @@
Url="/my/flask/demo"
AutheEnabled="#{ 32 + ${authUnauthenticated} }"
Description="Sample WSGI application using Flask"
- MatchRoles=":${dbrole}:%SQL:%Developer,%All:%Manager"
+ MatchRoles=":${dbrole}:%SQL:%Developer,%Manager:%All"
NameSpace="${ns}"
WSGIAppLocation="${libdir}flask-demo/flaskapp/"
WSGIAppName="app"
From e6afc10c07b4a96c28a3d706c25b58b8d141dd4d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 15:36:59 -0400
Subject: [PATCH 094/182] doc: minor change
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 5cb885bd..ac4d362f 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -221,7 +221,7 @@ Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private
Return $ListToString($lb("IPM", "readonly", ..ResourceReference.Module.Name, pDbResource), ".")
}
-Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
+Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
{
Set templateLists = $Listbuild("{$codeDbReadRole}", "${codeDbReadRole}")
Set role = "", ptr = 0 // We don't create the role unless ${codeDbReadRole} is explicitly in the MatchRoles
@@ -248,7 +248,7 @@ ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private
Return db.ResourceName
}
-/// Grant read permissions to the routine database. This is necessary because
+/// Grant read permissions to the routine database. This is necessary because routines and globals can have different DBs.
/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
{
From 070922d8421bc66ce48668aaea9bc1da855c816c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 16:09:59 -0400
Subject: [PATCH 095/182] enhance(webapp): use pkg name hash for role name to
avoid 64 char limit
---
src/cls/IPM/ResourceProcessor/WebApplication.cls | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index ac4d362f..f8641bc5 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -216,9 +216,17 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
{
- // Include the module name in the role name to avoid conflicts
+ // Include the module name in the role name to avoid conflicts, using hash because role name can't be longer than 64 characters
// This role should only be managed by IPM itself
- Return $ListToString($lb("IPM", "readonly", ..ResourceReference.Module.Name, pDbResource), ".")
+ Set hashBytes = $System.Encryption.MD5Hash(..ResourceReference.Module.Name) // 128 bits
+ Set hashHexString = $System.Encryption.Base32HexEncode(hashBytes, 1) // 26 characters, without '=' padding
+ // As long as pDbResource is shorter than 33 characters, the role name will be shorter than 64 characters
+ Set roleName = $ListToString($ListBuild("IPM", pDbResource, hashHexString), ".")
+ If $Length(roleName) > 64 {
+ // In the rare case where the hash is too long, we truncate the hash first
+ Set roleName = $Extract(roleName, 1, 64)
+ }
+ Return roleName
}
Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
@@ -274,7 +282,7 @@ Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %Strin
// This is the least-access means to be able to use the REST handler class
// in unauthenticated mode
Set roleName = ..ConstructRoleName(dbResource)
- Set roleProps("Description") = "Role created by IPM that grants only R permission on "_dbResource _ ". This role should only be managed by IPM."
+ Set roleProps("Description") = "Role created by IPM for package "_ ..ResourceReference.Module.Name _" that grants only R permission on "_dbResource _ ". This role should only be managed by IPM."
Set roleProps("Resources") = dbResource_":R"
Set roleProps("GrantedRoles") = ""
If ##class(Security.Roles).Exists(roleName) {
From c41eb7d05b4f845fea84208bd43a849d0ba2c5c6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 16:22:01 -0400
Subject: [PATCH 096/182] enhance: use numbers instead of Authe* macros to
avoid compile time errs
---
src/cls/IPM/Utils/Module.cls | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 985cfcb6..bbcc1d15 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2299,14 +2299,14 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set result = ..%RegExReplace(result, "dbrole", dbRole)
// Authentication bits used by Security.Applications
- Set result = ..%RegExReplace(result, "authK5API", $$$AutheK5API)
- Set result = ..%RegExReplace(result, "authPassword", $$$AutheCache)
- Set result = ..%RegExReplace(result, "authUnauthenticated", $$$AutheUnauthenticated)
- Set result = ..%RegExReplace(result, "authLDAP", $$$AutheLDAP)
- Set result = ..%RegExReplace(result, "authDelegated", $$$AutheDelegated)
- Set result = ..%RegExReplace(result, "authLoginToken", $$$AutheLoginToken)
- Set result = ..%RegExReplace(result, "authTwoFactorSMS", $$$AutheTwoFactorSMS)
- Set result = ..%RegExReplace(result, "authTwoFactorPW", $$$AutheTwoFactorPW)
+ Set result = ..%RegExReplace(result, "authK5API", 2**2)
+ Set result = ..%RegExReplace(result, "authPassword", 2**5)
+ Set result = ..%RegExReplace(result, "authUnauthenticated", 2**6)
+ Set result = ..%RegExReplace(result, "authLDAP", 2**11)
+ Set result = ..%RegExReplace(result, "authDelegated", 2**13)
+ Set result = ..%RegExReplace(result, "authLoginToken", 2**14)
+ Set result = ..%RegExReplace(result, "authTwoFactorSMS", 2**20)
+ Set result = ..%RegExReplace(result, "authTwoFactorPW", 2**21)
Return result
}
From 24d617b7e7db6bc13137ee41f625d4874169f34e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 16:25:23 -0400
Subject: [PATCH 097/182] docs: specify ${docDbReadRole} is meant for
WebApplication only
---
src/cls/IPM/Utils/Module.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index bbcc1d15..c0fff4e0 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2247,7 +2247,7 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
/// {$dbrole} - (deprecated in favor of {$globalsDbRole}) the current namespace's globals database role
/// {$globalsDbRole} - the current namespace's globals database role, functionally equivalent to {$dbrole} in legacy packages.
-/// {$codeDbReadRole} - the current namespace's routine database role (will be created if needed)
+/// {$codeDbReadRole} - [In WebApplication only] the current namespace's routine database role (will be created if needed)
/// {$authK5API} - the authentication macro $$$AutheK5API used for Security.Applications. Its value is equal to 4 (2^2).
/// {$authPassword} - the authentication macro $$$AutheCache used for Security.Applications. Its value is equal to 32 (2^5).
/// {$authLDAP} - the authentication macro $$$AutheLDAP used for Security.Applications. Its value is equal to 2048 (2^11).
From 68a0218e71e9577e5744c2cbefa82487996b7a4d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Sep 2024 16:27:37 -0400
Subject: [PATCH 098/182] feat: implement ${globalsDBRole} template
substitution
---
src/cls/IPM/Utils/Module.cls | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index c0fff4e0..509a5511 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2297,6 +2297,7 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set dbRole = ..GetDatabaseRole()
Set result = ..%RegExReplace(result, "dbrole", dbRole)
+ Set result = ..%RegExReplace(result, "globalsDbRole", dbRole)
// Authentication bits used by Security.Applications
Set result = ..%RegExReplace(result, "authK5API", 2**2)
From 3d47b4d4b3039b35b960437d994cdf5fc181a02c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 10:32:52 -0400
Subject: [PATCH 099/182] feat(webapp): simplify ${codeDbReadRole} without auto
deleting it
---
.../IPM/ResourceProcessor/WebApplication.cls | 37 +------------------
1 file changed, 1 insertion(+), 36 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index f8641bc5..1232feb6 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -88,7 +88,6 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
} ElseIf ((pPhase = "Unconfigure") || (pPhase = "Clean")) {
// Remove web application
Do ..DeleteWebApp(tVerbose)
- Do ..DeleteCodeReadRole($Get(..SecurityProperties("NameSpace")), tVerbose)
}
} Catch e {
Set sc = e.AsStatus()
@@ -216,17 +215,7 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
{
- // Include the module name in the role name to avoid conflicts, using hash because role name can't be longer than 64 characters
- // This role should only be managed by IPM itself
- Set hashBytes = $System.Encryption.MD5Hash(..ResourceReference.Module.Name) // 128 bits
- Set hashHexString = $System.Encryption.Base32HexEncode(hashBytes, 1) // 26 characters, without '=' padding
- // As long as pDbResource is shorter than 33 characters, the role name will be shorter than 64 characters
- Set roleName = $ListToString($ListBuild("IPM", pDbResource, hashHexString), ".")
- If $Length(roleName) > 64 {
- // In the rare case where the hash is too long, we truncate the hash first
- Set roleName = $Extract(roleName, 1, 64)
- }
- Return roleName
+ Return $ListToString($ListBuild("%DB", pDbResource, "READ"), "_")
}
Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
@@ -301,28 +290,4 @@ Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %Strin
Return roleName
}
-Method DeleteCodeReadRole(pNamespace As %String, pVerbose As %Boolean) [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- Set dbResource = ..GetDbResource(pNamespace)
- If pVerbose {
- Write !, "DB Resource: ", dbResource
- }
- Set roleName = ..ConstructRoleName(dbResource)
- If pVerbose {
- Write !, "Role to delete: ", roleName
- }
-
- If ##class(Security.Roles).Exists(roleName) {
- $$$ThrowOnError(##class(Security.Roles).Delete(roleName))
- If pVerbose {
- Write !, "Deleted succesfully"
- }
- } ElseIf pVerbose {
- Write !, "Skipped because role does not exist"
- }
-}
-
}
From 81c48a1ccc5dd08a085877b8b2804505dfe501fe Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 10:45:27 -0400
Subject: [PATCH 100/182] feat(webapp): remove ${codeDbReadRole} altogether
---
.../IPM/ResourceProcessor/WebApplication.cls | 82 -------------------
src/cls/IPM/Utils/Module.cls | 1 -
.../PM/Integration/InstallApplication.cls | 13 ---
.../PM/Integration/_data/rest-app/module.xml | 2 +-
.../PM/Integration/_data/rest-app2/module.xml | 36 --------
.../rest-app2/src/cls/Test/Rest/Demo2.cls | 17 ----
6 files changed, 1 insertion(+), 150 deletions(-)
delete mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
delete mode 100644 tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
diff --git a/src/cls/IPM/ResourceProcessor/WebApplication.cls b/src/cls/IPM/ResourceProcessor/WebApplication.cls
index 1232feb6..7199f7ed 100644
--- a/src/cls/IPM/ResourceProcessor/WebApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/WebApplication.cls
@@ -102,11 +102,6 @@ Method CreateOrUpdateWebApp(pVerbose As %Boolean = 0) [ Internal ]
Merge properties = ..SecurityProperties
- // Replace ${codeDbReadRole} with a role (potentially to be created) with read permission to routine database
- If ($Data(properties("NameSpace"), ns) # 2) && ($Data(properties("MatchRoles"), mr) # 2) {
- Set properties("MatchRoles") = ..ReplaceCodeReadRole(mr, ns, pVerbose)
- }
-
If ##class(Security.Applications).Exists(..Name) {
Write:pVerbose !,"Updating Web Application ",..Name
Set sc = ##class(Security.Applications).Get(..Name,.oldProperties)
@@ -213,81 +208,4 @@ ClassMethod DoPropertiesExist(pPropList As %Library.List) As %Library.List [ Int
Return badList
}
-Method ConstructRoleName(pDbResource As %String) As %String [ Internal, Private ]
-{
- Return $ListToString($ListBuild("%DB", pDbResource, "READ"), "_")
-}
-
-Method ReplaceCodeReadRole(pMatchRoles As %String, pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
-{
- Set templateLists = $Listbuild("{$codeDbReadRole}", "${codeDbReadRole}")
- Set role = "", ptr = 0 // We don't create the role unless ${codeDbReadRole} is explicitly in the MatchRoles
- While $ListNext(templateLists, ptr, template) {
- If pMatchRoles [ template {
- Set role = $Select(role="": ..CreateCodeReadRole(pNamespace, pVerbose), 1: role)
- Set pMatchRoles = $Replace(pMatchRoles, template, role)
- }
- }
- Return pMatchRoles
-}
-
-ClassMethod GetDbResource(pNamespace As %String) As %String [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- // Find the namespace-default routine DB
- $$$ThrowOnError(##class(Config.Namespaces).Get(pNamespace,.nsProps))
- Set defaultDB = nsProps("Routines")
- $$$ThrowOnError(##class(Config.Databases).Get(defaultDB,.dbProps))
- Set db=##Class(SYS.Database).%OpenId(dbProps("Directory"),,.sc)
- $$$ThrowOnError(sc)
- Return db.ResourceName
-}
-
-/// Grant read permissions to the routine database. This is necessary because routines and globals can have different DBs.
-/// Adapted from https://github.com/intersystems/isc-perf-ui/blob/main/cls/pkg/isc/perf/ui/Installer.cls
-Method CreateCodeReadRole(pNamespace As %String, pVerbose As %Boolean) As %String [ Internal, Private ]
-{
- New $Namespace
- Set $Namespace = "%SYS"
-
- set dbResource = ..GetDbResource(pNamespace)
- If pVerbose {
- Write !, "DB Resource: ", dbResource
- }
-
- // Is there public READ permission on the namespace-default routine DB?
- $$$ThrowOnError(##class(Security.Resources).Get(dbResource,.resourceProps))
- If (resourceProps("PublicPermission")#2) {
- // No need to do anything else if there's public R permission
- If pVerbose {
- Write !, "Public Read permission already exists on ", dbResource, !, "Skipping"
- }
- Return ""
- }
-
- // Create role granting R permission on namespace-default routine DB resource
- // This is the least-access means to be able to use the REST handler class
- // in unauthenticated mode
- Set roleName = ..ConstructRoleName(dbResource)
- Set roleProps("Description") = "Role created by IPM for package "_ ..ResourceReference.Module.Name _" that grants only R permission on "_dbResource _ ". This role should only be managed by IPM."
- Set roleProps("Resources") = dbResource_":R"
- Set roleProps("GrantedRoles") = ""
- If ##class(Security.Roles).Exists(roleName) {
- If pVerbose {
- Write !,"Updating role: ", roleName, " with properties: ", !
- Zwrite roleProps
- }
- $$$ThrowOnError(##class(Security.Roles).Modify(roleName,.roleProps))
- } Else {
- If pVerbose {
- Write !,"Creating role: ", roleName, " with properties: ", !
- Zwrite roleProps
- }
- $$$ThrowOnError(##class(Security.Roles).Create(roleName,roleProps("Description"),roleProps("Resources"),roleProps("GrantedRoles")))
- }
- Return roleName
-}
-
}
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 509a5511..491308f3 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2247,7 +2247,6 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
/// {$dbrole} - (deprecated in favor of {$globalsDbRole}) the current namespace's globals database role
/// {$globalsDbRole} - the current namespace's globals database role, functionally equivalent to {$dbrole} in legacy packages.
-/// {$codeDbReadRole} - [In WebApplication only] the current namespace's routine database role (will be created if needed)
/// {$authK5API} - the authentication macro $$$AutheK5API used for Security.Applications. Its value is equal to 4 (2^2).
/// {$authPassword} - the authentication macro $$$AutheCache used for Security.Applications. Its value is equal to 32 (2^5).
/// {$authLDAP} - the authentication macro $$$AutheLDAP used for Security.Applications. Its value is equal to 2048 (2^11).
diff --git a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
index 7dce3885..3e0357ab 100644
--- a/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
+++ b/tests/integration_tests/Test/PM/Integration/InstallApplication.cls
@@ -59,19 +59,6 @@ Method TestRestApp()
Do $$$AssertStatusOK(tSC,"Loaded REST Application ""restdemo/"" successfully. " _ tModuleDir1)
Do $$$AssertStatusOK(..HttpGet("restdemo/"))
- // Install another REST application of the same database
- // Uninstalling this module shouldn't affect the first one (beacuse ${codeDbReadRole} is unique for each module)
- Set tModuleDir2 = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/rest-app2/")
- Set tSC = ##class(%IPM.Main).Shell("load -verbose " _ tModuleDir2)
- Do $$$AssertStatusOK(tSC,"Loaded REST Application ""restdemo2/"" successfully. " _ tModuleDir2)
- Do $$$AssertStatusOK(tSC,"Loaded REST Application successfully. " _ tModuleDir2)
- Do $$$AssertStatusOK(..HttpGet("restdemo2/"))
-
- // Delete the 2nd REST application, the 1st one should still be accessible
- Set tSC = ##class(%IPM.Main).Shell("uninstall -verbose rest-demo2")
- Do $$$AssertStatusOK(tSC,"Deleted REST Application ""restdemo2/"" successfully. " _ tModuleDir2)
- Do $$$AssertStatusOK(..HttpGet("restdemo/"))
-
// Delete the 1st REST application
Set tSC = ##class(%IPM.Main).Shell("uninstall -verbose rest-demo")
Do $$$AssertStatusOK(tSC,"Deleted REST Application ""restdemo/"" successfully. " _ tModuleDir1)
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 7c21806a..97c83c4b 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -23,7 +23,7 @@
Path="/src"
Recurse="1"
Directory="{$cspdir}/restdemo"
- MatchRoles=":${dbrole}:%SQL:${codeDbReadRole}:%Developer,%Manager:%All"
+ MatchRoles=":${dbrole}:%SQL:%All:%Developer,%Manager:%All"
AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
deleted file mode 100644
index 4a76f620..00000000
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/module.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-
-
-
-
- rest-demo2
- 1.0.0
- This is a demo of a rest application
- rest
-
- Shuheng Liu
- InterSystems
- 2024
- MIT
- notes
-
- module
- src
-
-
-
- Module installed successfully!
-
-
-
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls b/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
deleted file mode 100644
index 8aa974a2..00000000
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app2/src/cls/Test/Rest/Demo2.cls
+++ /dev/null
@@ -1,17 +0,0 @@
-Class Test.Rest.Demo2 Extends %CSP.REST
-{
-
-XData UrlMap [ XMLNamespace = "http://www.intersystems.com/urlmap" ]
-{
-
-
-
-}
-
-ClassMethod GetInfo() As %Status
-{
- Write "Hello, World!"
- Quit $$$OK
-}
-
-}
From e28bd56942ba83a117ba8c969b92064034c59e4a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 14:42:53 -0400
Subject: [PATCH 101/182] feat: add support $$$macro in module.xml
---
src/cls/IPM/Storage/Module.cls | 1 +
src/cls/IPM/Utils/Module.cls | 36 ++++++++++++++++++++++++++++++++++
2 files changed, 37 insertions(+)
diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls
index 2992aca7..d2b7f71e 100644
--- a/src/cls/IPM/Storage/Module.cls
+++ b/src/cls/IPM/Storage/Module.cls
@@ -1427,6 +1427,7 @@ Method %Evaluate(pAttrValue As %String, ByRef pParams) As %String [ Internal ]
Set customParams("packagename") = ..Name
Set customParams("version") = ..VersionString
Set customParams("verbose") = +$Get(pParams("Verbose"))
+ Set tAttrValue = ##class(%IPM.Utils.Module).%EvaluateMacro(tAttrValue)
Set tAttrValue = ##class(%IPM.Storage.ModuleSetting.Default).EvaluateAttribute(tAttrValue,.customParams)
Set attrValue = ##class(%IPM.Utils.Module).%EvaluateSystemExpression(tAttrValue)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 491308f3..861098ad 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2311,6 +2311,42 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Return result
}
+ClassMethod %EvaluateMacro(pString As %String) As %String [ Internal ]
+{
+ Set tString = pString
+ Set tPattern = "\$\$\$[a-zA-Z0-9]+"
+ Set tMatch = ##class(%Regex.Matcher).%New(tPattern, tString)
+ While tMatch.Locate() {
+ Set group = tMatch.Group
+ Set macroVal = ..EvaluateMacroHelper($Extract(group, 4, *))
+ set tString = $Replace(tString, group, macroVal)
+ }
+ Return tString
+}
+
+/// Helper metohd to evaluate macro at runtime.
+ClassMethod EvaluateMacroHelper(pMacro As %String) As %String [ Internal, Private ]
+{
+ Set stream = ##class(%Stream.TmpCharacter).%New()
+ Do stream.WriteLine("Class %IPM.Generated.MacroEval")
+ Do stream.WriteLine("{")
+ Do stream.WriteLine("ClassMethod EvalMacro() [ CodeMode = objectgenerator ]")
+ Do stream.WriteLine("{")
+ Do stream.WriteLine(" Do %code.WriteLine(""#IfDef "_ pMacro _""")")
+ Do stream.WriteLine(" Do %code.WriteLine("" Quit $$$"_pMacro_""")")
+ Do stream.WriteLine(" Do %code.WriteLine(""#Else "")")
+ Do stream.WriteLine(" Do %code.WriteLine("" Quit """""""" "")")
+ Do stream.WriteLine(" Do %code.WriteLine(""#EndIf "")")
+ Do stream.WriteLine(" Quit $$$OK")
+ Do stream.WriteLine("}")
+ Do stream.WriteLine("}")
+
+ Do $system.OBJ.LoadStream(stream, "ck-d")
+ Set value = $ClassMethod("%IPM.Generated.MacroEval", "EvalMacro")
+ Do $system.OBJ.Delete("%IPM.Generated.MacroEval", "-d")
+ Return value
+}
+
/// Wrapper around Config.Namespaces:Get()
ClassMethod GetDatabaseInfoForNamespace(pNamespace As %String, Output pProperties) [ Internal ]
{
From 59c94b40f074574cfadd8ab5886d5babdf1dbdd9 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 14:43:17 -0400
Subject: [PATCH 102/182] test(macro): test expansion of $$$macro in string
---
tests/unit_tests/Test/PM/Unit/Module.cls | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/tests/unit_tests/Test/PM/Unit/Module.cls b/tests/unit_tests/Test/PM/Unit/Module.cls
index b3269d9d..4bd5d1ef 100644
--- a/tests/unit_tests/Test/PM/Unit/Module.cls
+++ b/tests/unit_tests/Test/PM/Unit/Module.cls
@@ -27,4 +27,12 @@ Method TestEvaluateAttribute()
Do $$$AssertEquals(tModule.%Evaluate("#{..Root}"),tModule.Root)
}
+Method TestEvaluateMacro()
+{
+ Do $$$AssertEquals(##class(%IPM.Utils.Module).%EvaluateMacro("$$$ "), "$$$ ")
+ Do $$$AssertEquals(##class(%IPM.Utils.Module).%EvaluateMacro("Hello $$$OK $$$RandomJibberish123 World"), "Hello 1 World")
+ Do $$$AssertEquals(##class(%IPM.Utils.Module).%EvaluateMacro("Hello $$$OK$$$RandomJibberish123 World"), "Hello 1 World")
+ Do $$$AssertEquals(##class(%IPM.Utils.Module).%EvaluateMacro("$$$AutheUnauthenticated + $$$AutheCache + MAGIC"), $$$AutheUnauthenticated _ " + " _ $$$AutheCache _ " + MAGIC")
+}
+
}
From 047e6cd86028ed362217ac981f88472fc7478099 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 14:54:08 -0400
Subject: [PATCH 103/182] docs: add docs for $$$macro expansion
---
CHANGELOG.md | 1 +
src/cls/IPM/Utils/Module.cls | 8 +-------
2 files changed, 2 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1e20eb23..865b12a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #518 Added ability to show source file location when running `list-installed`. E.g., `zpm "list-installed -showsource"`.
- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
- #562 Added a generic resource processpor `WebApplication`, which handles creating and removal of all Security.Applications resources
+- #575 Added ability to expand `$$$macro` in module.xml. The macro cannot take any arguments yet.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 861098ad..bcf7c41d 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2247,13 +2247,7 @@ ClassMethod GetFileLines(pFileName As %String, Output pOutput) As %Status [ Inte
/// {$webroot} - the instance's constructed url with host and port (e.g. http://123.45.678.90:52773/)
/// {$dbrole} - (deprecated in favor of {$globalsDbRole}) the current namespace's globals database role
/// {$globalsDbRole} - the current namespace's globals database role, functionally equivalent to {$dbrole} in legacy packages.
-/// {$authK5API} - the authentication macro $$$AutheK5API used for Security.Applications. Its value is equal to 4 (2^2).
-/// {$authPassword} - the authentication macro $$$AutheCache used for Security.Applications. Its value is equal to 32 (2^5).
-/// {$authLDAP} - the authentication macro $$$AutheLDAP used for Security.Applications. Its value is equal to 2048 (2^11).
-/// {$authDelegated} - the authentication macro $$$AutheDelegated used for Security.Applications. Its value is equal to 8192 (2^13).
-/// {$authLoginToken} - the authentication macro $$$AutheLogintToken used for Security.Applications. Its value is equal to 16384 (2^14).
-/// {$authTwoFactorSMS} - the authentication macro $$$AutheTwoFactorSMS used for Security.Applications. Its value is equal to 1048576 (2^20).
-/// {$authTwoFactorPW} - the authentication macro $$$AutheTwoFactorPW used for Security.Applications. Its value is equal to 2097152 (2^21).
+/// $$$macros - triple dollars ($$$) followed immediately by alphanumeric string will be substituted with corresponding macro value. If the macro is undefined on the instance, will return null ("")
///
/// For backward compatibility, supporting both ${var} and {$var}
ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal ]
From 146b852e4d875c9bb9f113ad54366d28ea138f91 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 14:59:15 -0400
Subject: [PATCH 104/182] enhance(webapp): use $$$macro expansion instead of
${auth*} templates
---
src/cls/IPM/ResourceProcessor/Abstract.cls | 1 +
src/cls/IPM/Utils/Module.cls | 10 ----------
.../Test/PM/Integration/_data/rest-app/module.xml | 2 +-
3 files changed, 2 insertions(+), 11 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/Abstract.cls b/src/cls/IPM/ResourceProcessor/Abstract.cls
index 9b596b8d..ac06e006 100644
--- a/src/cls/IPM/ResourceProcessor/Abstract.cls
+++ b/src/cls/IPM/ResourceProcessor/Abstract.cls
@@ -252,6 +252,7 @@ Method %Evaluate(pAttrValue As %String) As %String [ Internal ]
If (tAttrValue '[ "{") {
Return tAttrValue
}
+ Set tAttrValue = ##class(%IPM.Utils.Module).%EvaluateMacro(tAttrValue)
Set attrValue = ##class(%IPM.Utils.Module).%EvaluateSystemExpression(tAttrValue)
Set root = ..ResourceReference.Module.Root
If (root '= "") {
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index bcf7c41d..290cf596 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2292,16 +2292,6 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Set result = ..%RegExReplace(result, "dbrole", dbRole)
Set result = ..%RegExReplace(result, "globalsDbRole", dbRole)
- // Authentication bits used by Security.Applications
- Set result = ..%RegExReplace(result, "authK5API", 2**2)
- Set result = ..%RegExReplace(result, "authPassword", 2**5)
- Set result = ..%RegExReplace(result, "authUnauthenticated", 2**6)
- Set result = ..%RegExReplace(result, "authLDAP", 2**11)
- Set result = ..%RegExReplace(result, "authDelegated", 2**13)
- Set result = ..%RegExReplace(result, "authLoginToken", 2**14)
- Set result = ..%RegExReplace(result, "authTwoFactorSMS", 2**20)
- Set result = ..%RegExReplace(result, "authTwoFactorPW", 2**21)
-
Return result
}
diff --git a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
index 97c83c4b..0d46be0a 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/rest-app/module.xml
@@ -24,7 +24,7 @@
Recurse="1"
Directory="{$cspdir}/restdemo"
MatchRoles=":${dbrole}:%SQL:%All:%Developer,%Manager:%All"
- AutheEnabled="#{${authPassword} + ${authUnauthenticated}}"
+ AutheEnabled="#{$$$AutheCache + $$$AutheUnauthenticated}"
DispatchClass="Test.Rest.Demo"
ServeFiles="1"
CookiePath="/restdemo"
From f9bc432a57a8d1b1fce01c03c68553b6e19fa671 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 19 Sep 2024 15:20:41 -0400
Subject: [PATCH 105/182] enhance: improve error handling and efficiency for
macro expansion
---
src/cls/IPM/Utils/Module.cls | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 290cf596..5cec60ba 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -2297,6 +2297,9 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
ClassMethod %EvaluateMacro(pString As %String) As %String [ Internal ]
{
+ If pString '[ "$$$" {
+ Return pString // Return early for efficiency
+ }
Set tString = pString
Set tPattern = "\$\$\$[a-zA-Z0-9]+"
Set tMatch = ##class(%Regex.Matcher).%New(tPattern, tString)
@@ -2325,7 +2328,9 @@ ClassMethod EvaluateMacroHelper(pMacro As %String) As %String [ Internal, Privat
Do stream.WriteLine("}")
Do stream.WriteLine("}")
- Do $system.OBJ.LoadStream(stream, "ck-d")
+ // If it reaches here, pMacro is a valid expression (although may be undefined).
+ // We should throw an error instead of failing silently.
+ $$$ThrowOnError($system.OBJ.LoadStream(stream, "ck-d"))
Set value = $ClassMethod("%IPM.Generated.MacroEval", "EvalMacro")
Do $system.OBJ.Delete("%IPM.Generated.MacroEval", "-d")
Return value
From 7a00208f76fbfe6ad317c39da77df14853ff8291 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 20 Sep 2024 10:43:48 -0400
Subject: [PATCH 106/182] style: misc style change based on code review
feedback
---
src/cls/IPM/Main.cls | 32 ++++++++++++++++++++++++++++----
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 5c4882dd..08dbbeb1 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -1156,6 +1156,20 @@ ClassMethod Version(ByRef pCommandInfo) [ Internal ]
Quit $$$OK
}
+/// Argument `list` is a multidimensional array where `list` contains the a numeric value `n`, indicating the length of the list
+/// Each subnode `list(i)` where 1 <= i <= n contains a $ListBuild of a single namespace
+/// Each subnote `list(i,"modules")` contains the number of modules in the namespace
+/// For 1 <= j <= `list(i,"modules")`, each subnode `list(i,"modules",j)` contains a $ListBuild of the module name, version, external name, and whether the module is installed in developer mode
+/// Additional nodes are also accepted, although they are not used in this method
+/// For example:
+/// list=1
+/// list(1)=$lb("USER")
+/// list(1,"modules")=2
+/// list(1,"modules",1)=$lb("zpm","0.9.0-SNAPSHOT","Package Management System",1)
+/// list(1,"modules",2)=$lb("zpm-registry","1.3.2","ZPM Registry",0)
+/// list(1,"modules","width")=12
+/// list("ns")="*"
+/// list("width")=4
ClassMethod AddIPMMappedNamespaces(ByRef list) [ Private ]
{
Do ..GetListNamespace(.allNS,"*")
@@ -1172,7 +1186,9 @@ ClassMethod AddIPMMappedNamespaces(ByRef list) [ Private ]
Set namespace = ""
For {
Set namespace = $Order(allNS(namespace))
- Quit:namespace=""
+ If namespace = "" {
+ Quit
+ }
Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(namespace,"%IPM")
If $Data(mappedFrom(sourceDB),sourceNamespace) {
@@ -2680,11 +2696,14 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
If globally {
Set namespaces = ""
Do ..GetListNamespace(.list)
+ // Namespace-specific mappings aren't allowed to override %ALL mappings. See https://github.com/intersystems/ipm/pull/485
Kill list("%ALL")
Set key = ""
For {
Set key = $Order(list(key))
- Quit:key=""
+ If key = "" {
+ Quit
+ }
Set namespaces = namespaces _ $ListBuild(key)
}
} Else {
@@ -2692,8 +2711,8 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
}
Set pointer = 0
While $ListNext(namespaces,pointer,namespace) {
- Set $Namespace = namespace
- If $$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName) {
+ Set $Namespace = $Zstrip(namespace, "<>WC")
+ If ..IPMInstalled() {
If 'quiet || preview {
Write !,"Skipping "_namespace_" - IPM already installed."
}
@@ -2926,6 +2945,11 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
}
}
+ClassMethod IPMInstalled() As %Boolean [ CodeMode = expression ]
+{
+$$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName)
+}
+
/// Runs package manager commands in a way that is friendly to the OS-level shell.
/// Creates pOutputLogFile if it does not exist.
/// If it does, and pAppendToLog is true, appends to it; otherwise, deletes the file before outputting to it.
From bf8ed87be9f4abab369e93b217d5e42dc5d23bc6 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 20 Sep 2024 10:53:44 -0400
Subject: [PATCH 107/182] docs(main): remove inaccurate description of mapping
to %ALL
---
src/cls/IPM/Main.cls | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 08dbbeb1..9f336723 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -614,11 +614,11 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
-
+
enable -map -globally
From a9e8cbbdfb352332aca760d5e77d6c204be0ab70 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 23 Sep 2024 10:48:58 -0400
Subject: [PATCH 108/182] fix: revert changes in MapOnly XData and Map Method
---
preload/cls/IPM/Installer.cls | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 5a5a7200..17d27074 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -44,10 +44,10 @@ XData PM [ XMLNamespace = INSTALLER ]
XData MapOnly [ XMLNamespace = INSTALLER ]
{
-
+
-
+
@@ -80,6 +80,7 @@ ClassMethod Map(ByRef pVars, pLogLevel As %Integer, pInstaller As %Installer.Ins
Do %code.WriteLine(" Set sc = ##class(Config.Namespaces).Get(prevNS, .properties)")
Do %code.WriteLine(" Set pVars(""CURNSRoutineDB"")=$Get(properties(""Routines""))")
Do %code.WriteLine(" Set pVars(""CURNSGlobalDB"")=$Get(properties(""Globals""))")
+ Do %code.WriteLine(" If ##class(Config.MapPackages).Exists(prevNS,""%IPM"") { Quit $$$OK }")
Quit ##class(%Installer.Manifest).%Generate(%compiledclass, %code, "MapOnly")
}
From 0a6887e10cce0c1a36edd1242947b16bd3b6d842 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 24 Sep 2024 11:25:13 -0400
Subject: [PATCH 109/182] docs(enable): specify when `-globally` tag includes
%SYS
---
src/cls/IPM/Main.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 9f336723..1a4cb564 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -611,7 +611,7 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
From 0ce3be4048385e863a82cc84961e59cda7ea55c7 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 24 Sep 2024 11:28:47 -0400
Subject: [PATCH 110/182] enhance: show ns-specific IPM versions when calling
zpm from ns w/o IPM
---
src/cls/IPM/Main.cls | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 1a4cb564..c1f31bee 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -2648,8 +2648,25 @@ ZPM(pArgs...)
If $System.CLS.IsMthd("%IPM.Main", "Shell") {
Do ##class(%IPM.Main).Shell(pArgs...) Quit
} Else {
- // TODO: Decide what to do here; definitely needs to support enabling IPM from here as well,
- // but need to decide what level of customization to provide
+ // TODO: Needs to support enabling IPM from here, also need to decide what level of customization to provide
+ Write !, "IPM is not enabled in this namespace."
+ &sql(SELECT %DLIST(Nsp) INTO :namespaces FROM %SYS.Namespace_List())
+ If SQLCODE '= 0 {
+ Write !, "Error getting namespace name. SQLCODE =", SQLCODE
+ } Else {
+ New $Namespace
+ Set ptr = 0
+ While $ListNext(namespaces, ptr, ns) {
+ Set $Namespace = $Zstrip(ns, "<>WC")
+ If $System.CLS.IsMthd("%IPM.Main", "Shell") {
+ Write !, "Change namepace to one of the following to run the ""zpm"" command"
+ Do ##class(%IPM.Main).Shell("version")
+ Return ""
+ }
+ }
+ // Shouldn't happen since %ZLANGC00.mac and %ZLANGF00.mac are present
+ Write "No namespace found with IPM enabled."
+ }
Quit ""
}
}
From 906423ea957590e60371ec59fffe6a76e31999c8 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 24 Sep 2024 14:26:26 -0400
Subject: [PATCH 111/182] fix: map IPM to all namespaces when upgrading from
ZPM via registry
---
module.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/module.xml b/module.xml
index 2a632ba1..45e95440 100644
--- a/module.xml
+++ b/module.xml
@@ -19,6 +19,7 @@
+
${verbose}
From faf4ebdca219422074b7a51c0b8fb1549aec5492 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 25 Sep 2024 15:33:21 -0400
Subject: [PATCH 112/182] enhance: disallow management of IPM if mapped from
another namespace
---
src/cls/IPM/Main.cls | 46 +++++++++++++++++++++++++++++++++++++++++---
1 file changed, 43 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index c1f31bee..9e146b4e 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -1925,6 +1925,35 @@ ClassMethod Load(ByRef pCommandInfo) [ Internal ]
}
}
+ClassMethod CheckModuleNamespace(pModuleName As %String) As %Status
+{
+ // ZPM can be mapped from another namespace. If so, we need to perform upgrade in the source namespace.
+ Do ..GetListModules("*", pModuleName, .list)
+ Do ..AddIPMMappedNamespaces(.list)
+ Set found = 0
+ Set affectedNamespace = ""
+ For i = 1:1:list {
+ If list(i) = $ListBuild($Namespace) {
+ Set found = 1
+ If $Data(list(i, "modules", 1, "Installed In"), mappedFrom) # 2 {
+ Write "Module "_pModuleName_" is mapped from namespace "_mappedFrom_" and must be managed from there.",!
+ Return $$$ERROR($$$GeneralError, $$$FormatText("Cannot manage '%1' in this namespace", pModuleName))
+ }
+ } ElseIf $Get(list(i, "modules", 1, "Installed In")) = $Namespace {
+ Set affectedNamespace = affectedNamespace _ list(i)
+ }
+ }
+ If 'found {
+ // This shouldn't happen, but worth checking for.
+ $$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("Could not find namespace for '%1'", pModuleName)))
+ }
+ If affectedNamespace '= "" {
+ Set affectedNamespace = $ListBuild($Namespace) _ affectedNamespace
+ Write !, "The following namespaces fill be affected: ", $LISTTOSTRING(affectedNamespace, ", ")
+ }
+ Quit $$$OK
+}
+
ClassMethod Install(ByRef pCommandInfo) [ Internal ]
{
Set tRegistry = ""
@@ -1932,9 +1961,13 @@ ClassMethod Install(ByRef pCommandInfo) [ Internal ]
If (tModuleName["/") {
Set $ListBuild(tRegistry, tModuleName) = $ListFromString(tModuleName, "/")
}
- If (tModuleName = "") {
- Quit $$$OK
- }
+ If (tModuleName = "") {
+ Quit $$$OK
+ }
+
+ If tModuleName = $$$IPMModuleName {
+ $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ }
Set tVersion = $Get(pCommandInfo("parameters","version"))
Set tKeywords = $$$GetModifier(pCommandInfo,"keywords")
@@ -2018,6 +2051,10 @@ ClassMethod Reinstall(ByRef pCommandInfo) [ Internal ]
$$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("Module '%1' is not currently installed.",tModuleName)))
}
$$$ThrowOnError(tSC)
+
+ If tModuleName = $$$IPMModuleName {
+ $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ }
Set tVersionString = tModule.Version.ToString()
Write !,"Reinstalling ",tModuleName," ",tVersionString
@@ -2034,6 +2071,9 @@ ClassMethod Uninstall(ByRef pCommandInfo) [ Internal ]
Return
} Else {
Set tModuleName = pCommandInfo("parameters","module")
+ If tModuleName = $$$IPMModuleName {
+ $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ }
Set tRecurse = $$$HasModifier(pCommandInfo,"recurse") // Recursively uninstall unneeded dependencies
$$$ThrowOnError(##class(%IPM.Storage.Module).Uninstall(tModuleName,tForce,tRecurse,.tParams))
}
From bf6e69973c179d3d55e59d9309c440f1510962a9 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 25 Sep 2024 16:19:30 -0400
Subject: [PATCH 113/182] fix: perform migration in all namespaces when
upgrading from ZPM to IPM
---
src/cls/IPM/Utils/Migration.cls | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 520f5bb5..c20128c2 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -7,7 +7,17 @@ ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
Set sc = $$$OK
Try {
- Do ..MigrateZPMToIPM(verbose)
+ New $Namespace
+ Do ##class(%IPM.Main).GetListNamespace(.list)
+ set ns = ""
+ For {
+ Set ns = $Order(list(ns))
+ If ns = "" {
+ Quit
+ }
+ Set $Namespace = $Zstrip(ns,"<>WC")
+ Do ..MigrateZPMToIPM(verbose)
+ }
} Catch e {
Set sc = e.AsStatus()
}
@@ -22,6 +32,9 @@ ClassMethod HasLegacyZPMPackage()
ClassMethod MigrateZPMToIPM(verbose As %Boolean = 1)
{
+ If verbose {
+ Write !, "Migrating ZPM data to IPM... in namespace ", $Namespace
+ }
If '..HasLegacyZPMPackage() {
Write:verbose !,"Older IPM version not found; nothing to migrate.",!
Quit
From 1de4bd0bebaff07d5f01975cc37504950a39c34a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 25 Sep 2024 17:25:19 -0400
Subject: [PATCH 114/182] feat: implement an "unmap" command to allow for
per-NS upgrade of IPM
---
src/cls/IPM/Main.cls | 125 ++++++++++++++++++++++++++++++++++---------
1 file changed, 100 insertions(+), 25 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 9e146b4e..ad108d73 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -635,6 +635,21 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
+
+
+ Unmap %IPM package and rountines in specified namespaces. Will Skip non-mapped namespaces.
+
+
+
+
+
+ unmap -ns NS1,NS2,NS3
+
+
+ unmap -g
+
+
+
}
@@ -804,6 +819,8 @@ ClassMethod ShellInternal(pCommand As %String, Output pException As %Exception.A
Do ..Namespace(.tCommandInfo)
} ElseIf (tCommandInfo = "enable") {
Do ..EnableIPM(.tCommandInfo)
+ } ElseIf (tCommandInfo = "unmap") {
+ Do ..UnmapIPM(.tCommandInfo)
}
} Catch pException {
If (pException.Code = $$$ERCTRLC) {
@@ -1925,32 +1942,23 @@ ClassMethod Load(ByRef pCommandInfo) [ Internal ]
}
}
-ClassMethod CheckModuleNamespace(pModuleName As %String) As %Status
+ClassMethod CheckModuleNamespace() As %Status
{
- // ZPM can be mapped from another namespace. If so, we need to perform upgrade in the source namespace.
- Do ..GetListModules("*", pModuleName, .list)
- Do ..AddIPMMappedNamespaces(.list)
- Set found = 0
- Set affectedNamespace = ""
- For i = 1:1:list {
- If list(i) = $ListBuild($Namespace) {
- Set found = 1
- If $Data(list(i, "modules", 1, "Installed In"), mappedFrom) # 2 {
- Write "Module "_pModuleName_" is mapped from namespace "_mappedFrom_" and must be managed from there.",!
- Return $$$ERROR($$$GeneralError, $$$FormatText("Cannot manage '%1' in this namespace", pModuleName))
+ Do ..GetMapInfo(.tIsMappedFrom, .tIsMappedTo)
+ If $Data(tIsMappedFrom($Namespace), sourceNS) # 2 {
+ Quit $$$ERROR($$$GeneralError, $$$FormatText("Cannot install '%1' in namespace '%2' because it is mapped from namespace '%3'.", $$$IPMModuleName, $Namespace, sourceNS))
+ }
+ If $Data(tIsMappedTo($Namespace)) / 2 {
+ Write !, "Will affect the following namespaces:", $NAMESPACE
+ Set ns = ""
+ For {
+ Set ns = $Order(tIsMappedTo($Namespace, ns))
+ If ns = "" {
+ Quit
}
- } ElseIf $Get(list(i, "modules", 1, "Installed In")) = $Namespace {
- Set affectedNamespace = affectedNamespace _ list(i)
+ Write ", ", ns
}
}
- If 'found {
- // This shouldn't happen, but worth checking for.
- $$$ThrowStatus($$$ERROR($$$GeneralError,$$$FormatText("Could not find namespace for '%1'", pModuleName)))
- }
- If affectedNamespace '= "" {
- Set affectedNamespace = $ListBuild($Namespace) _ affectedNamespace
- Write !, "The following namespaces fill be affected: ", $LISTTOSTRING(affectedNamespace, ", ")
- }
Quit $$$OK
}
@@ -1966,7 +1974,7 @@ ClassMethod Install(ByRef pCommandInfo) [ Internal ]
}
If tModuleName = $$$IPMModuleName {
- $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ $$$ThrowOnError(..CheckModuleNamespace())
}
Set tVersion = $Get(pCommandInfo("parameters","version"))
@@ -2053,7 +2061,7 @@ ClassMethod Reinstall(ByRef pCommandInfo) [ Internal ]
$$$ThrowOnError(tSC)
If tModuleName = $$$IPMModuleName {
- $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ $$$ThrowOnError(..CheckModuleNamespace())
}
Set tVersionString = tModule.Version.ToString()
@@ -2072,7 +2080,7 @@ ClassMethod Uninstall(ByRef pCommandInfo) [ Internal ]
} Else {
Set tModuleName = pCommandInfo("parameters","module")
If tModuleName = $$$IPMModuleName {
- $$$ThrowOnError(..CheckModuleNamespace(tModuleName))
+ $$$ThrowOnError(..CheckModuleNamespace())
}
Set tRecurse = $$$HasModifier(pCommandInfo,"recurse") // Recursively uninstall unneeded dependencies
$$$ThrowOnError(##class(%IPM.Storage.Module).Uninstall(tModuleName,tForce,tRecurse,.tParams))
@@ -3002,6 +3010,73 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
}
}
+ClassMethod UnmapIPM(ByRef pCommandInfo)
+{
+ Set globally = $$$HasModifier(pCommandInfo,"globally")
+ Set namespaces = $ListFromString($$$GetModifier(pCommandInfo,"namespaces"), ",")
+ Set verbose = '$$$HasModifier(pCommandInfo,"quiet")
+ // Sanity check
+ If (globally && (namespaces '= "")) {
+ $$$ThrowOnError($$$ERROR($$$GeneralError,"Cannot specify namespaces and global unmap flag at the same time."))
+ }
+ If ('globally && (namespaces = "")) {
+ $$$ThrowOnError($$$ERROR($$$GeneralError,"Must specify either -globally or a list of namespaces with -ns"))
+ }
+ // populate namespaces if globally is set
+ If globally {
+ Do ..GetListNamespace(.nsTree)
+ Set ns = ""
+ For {
+ Set ns = $Order(nsTree(ns))
+ If ns = "" {
+ Quit
+ }
+ Set namespaces = namespaces_$ListBuild(ns)
+ }
+ }
+ If verbose {
+ Write !,"Will attempt to unmap %IPM package and routines from: "_ $ListToString(namespaces, ", ")
+ }
+
+ // Gather namespaces where %IPM is mapped into
+ Do ..GetMapInfo(.tIsMappedFrom)
+
+ Set ptr = 0
+ While $ListNext(namespaces, ptr, ns) {
+ Set src = $Get(tIsMappedFrom(ns))
+ If (src = "") {
+ If verbose {
+ Write !,"No mapping found for "_ns_". Skipping."
+ }
+ Continue
+ }
+ If verbose {
+ Write !,"Unmapping %IPM package and routines from "_ns_" (mapped from "_src_")"
+ }
+ $$$ThrowOnError(##class(%IPM.Utils.Module).RemovePackageMapping(ns, "%IPM"))
+ $$$ThrowOnError(##class(%IPM.Utils.Module).RemoveRoutineMapping(ns, "%IPM.*"))
+ }
+}
+
+/// Get the mapping relationship for %IPM package. Format:
+///
+/// - pIsMappedFrom(destinationNamespace) = sourceNamespace
+/// - pIsMappedTo(sourceNamespace, destinationNamespace) = ""
+///
+ClassMethod GetMapInfo(Output pIsMappedFrom, Output pIsMappedTo)
+{
+ Kill pIsMappedFrom, pIsMappedTo
+ Do ..GetListModules("*", $$$IPMModuleName, .list)
+ Do ..AddIPMMappedNamespaces(.list)
+ For i = 1:1:list {
+ Set destNS = $ListGet(list(i))
+ If $Data(list(i, "modules", 1, "Installed In"), srcNS) # 2 {
+ Set pIsMappedFrom(destNS) = srcNS
+ Set pIsMappedTo(srcNS, destNS) = ""
+ }
+ }
+}
+
ClassMethod IPMInstalled() As %Boolean [ CodeMode = expression ]
{
$$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName)
From 6db52874a2ee9045f182849e92691535a76147f4 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 26 Sep 2024 10:03:27 -0400
Subject: [PATCH 115/182] fix: run migration in each namespace with its own
try-catch
---
src/cls/IPM/Utils/Migration.cls | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index c20128c2..9b8c4b3b 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -5,21 +5,21 @@ Class %IPM.Utils.Migration
ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
+ New $Namespace
Set sc = $$$OK
- Try {
- New $Namespace
- Do ##class(%IPM.Main).GetListNamespace(.list)
- set ns = ""
- For {
- Set ns = $Order(list(ns))
- If ns = "" {
- Quit
- }
+ Do ##class(%IPM.Main).GetListNamespace(.list)
+ Set ns = ""
+ For {
+ Set ns = $Order(list(ns))
+ If ns = "" {
+ Quit
+ }
+ Try {
Set $Namespace = $Zstrip(ns,"<>WC")
Do ..MigrateZPMToIPM(verbose)
+ } Catch e {
+ Set sc = $$$ADDSC(sc, e.AsStatus())
}
- } Catch e {
- Set sc = e.AsStatus()
}
Quit sc
}
From 62d4a9967524a18b847a5513ad00ad3e7f0a3f43 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 27 Sep 2024 10:19:47 -0400
Subject: [PATCH 116/182] doc: improve documentation of unmap
---
src/cls/IPM/Main.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index ad108d73..1fad23a6 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -640,7 +640,7 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
Unmap %IPM package and rountines in specified namespaces. Will Skip non-mapped namespaces.
-
+
unmap -ns NS1,NS2,NS3
From cc2e4f6ce46c89da32312eddb32229d04fa887f4 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 27 Sep 2024 10:20:26 -0400
Subject: [PATCH 117/182] enhance: improve error handling based on pArgs, e.g.
`zpm "cmd":1:1`
---
src/cls/IPM/Main.cls | 22 +++++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 1fad23a6..e50a6d7e 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -2697,6 +2697,8 @@ ZPM(pArgs...)
Do ##class(%IPM.Main).Shell(pArgs...) Quit
} Else {
// TODO: Needs to support enabling IPM from here, also need to decide what level of customization to provide
+ Set tQuitOnError = $Get(pArgs(2))
+ Set tHaltOnComplete = $Get(pArgs(3))
Write !, "IPM is not enabled in this namespace."
&sql(SELECT %DLIST(Nsp) INTO :namespaces FROM %SYS.Namespace_List())
If SQLCODE '= 0 {
@@ -2704,18 +2706,32 @@ ZPM(pArgs...)
} Else {
New $Namespace
Set ptr = 0
+ Set found = 0
While $ListNext(namespaces, ptr, ns) {
Set $Namespace = $Zstrip(ns, "<>WC")
If $System.CLS.IsMthd("%IPM.Main", "Shell") {
Write !, "Change namepace to one of the following to run the ""zpm"" command"
Do ##class(%IPM.Main).Shell("version")
- Return ""
+ Set found = 1
+ Quit
}
}
// Shouldn't happen since %ZLANGC00.mac and %ZLANGF00.mac are present
- Write "No namespace found with IPM enabled."
+ If 'found {
+ Write !, "No namespace found with IPM enabled."
+ }
+ }
+
+ If tQuitOnError {
+ Write ! // Add a newline before quitting, otherwise the shell prompt will be on the same line as the error message
+ Do $System.Process.Terminate($Job, 1)
+ }
+ If tHaltOnComplete {
+ Write ! // Add a newline before halting
+ Halt
}
- Quit ""
+ // The $$$ERROR macro is not available for use in language extension
+ Return $$Error^%apiOBJ(5001,"IPM is not enabled in this namespace.")
}
}
From 7b10d493aad89bdbc3e4d91d2ff0ceb5ffcff671 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 27 Sep 2024 11:55:23 -0400
Subject: [PATCH 118/182] fix: only perform migration in NS with %IPM mapped
from current NS
---
src/cls/IPM/Utils/Migration.cls | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 9b8c4b3b..32943a33 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -5,12 +5,20 @@ Class %IPM.Utils.Migration
ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
+ Set tOriginalNS = $Namespace
New $Namespace
- Set sc = $$$OK
+ Set $Namespace = "%SYS"
+ Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(tOriginalNS, "%IPM")
+
Do ##class(%IPM.Main).GetListNamespace(.list)
Set ns = ""
+ Set sc = $$$OK
For {
Set ns = $Order(list(ns))
+ // Perform migration for namespaces to which %IPM is mapped from the current namespace's default routine database
+ If ##class(%SYS.Namespace).GetPackageDest(ns, "%IPM") '= sourceDB {
+ Continue
+ }
If ns = "" {
Quit
}
From 07781667952f8b55866b097e9c173c3a750c8a30 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 27 Sep 2024 12:01:03 -0400
Subject: [PATCH 119/182] fix: run %IPM.Main in the correct NS
---
src/cls/IPM/Utils/Migration.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 32943a33..6baf4a45 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -6,11 +6,11 @@ Class %IPM.Utils.Migration
ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
Set tOriginalNS = $Namespace
+ Do ##class(%IPM.Main).GetListNamespace(.list)
New $Namespace
Set $Namespace = "%SYS"
Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(tOriginalNS, "%IPM")
- Do ##class(%IPM.Main).GetListNamespace(.list)
Set ns = ""
Set sc = $$$OK
For {
From 3a6a5ef0fc2980afc471277ce166d2bb1c4de91c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 27 Sep 2024 13:38:13 -0400
Subject: [PATCH 120/182] fix: should check repo (instead of package) works in
MigrateOneRepo()
---
src/cls/IPM/Utils/Migration.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 6baf4a45..95fc23af 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -223,7 +223,7 @@ ClassMethod MigrateOneRepo(oldId As %String, name As %String, verbose As %Boolea
Merge ^IPM.Repo.DefinitionD(newId) = data
// Make sure loading/saving the object works
- Set newObj = ##class(%IPM.Storage.Module).%OpenId(newId,,.sc)
+ Set newObj = ##class(%IPM.Repo.Definition).%OpenId(newId,,.sc)
$$$ThrowOnError(sc)
// Save object to validate
From ed9589da86155ea4bc8d7fe3297f8e385da31173 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 09:07:24 -0400
Subject: [PATCH 121/182] style: typo and style fixes
---
src/cls/IPM/Main.cls | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index e50a6d7e..3eaf0784 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -613,7 +613,7 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
@@ -637,7 +637,7 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
- Unmap %IPM package and rountines in specified namespaces. Will Skip non-mapped namespaces.
+ Unmap %IPM package and routines in specified namespaces. Will Skip non-mapped namespaces.
@@ -2697,8 +2697,8 @@ ZPM(pArgs...)
Do ##class(%IPM.Main).Shell(pArgs...) Quit
} Else {
// TODO: Needs to support enabling IPM from here, also need to decide what level of customization to provide
- Set tQuitOnError = $Get(pArgs(2))
- Set tHaltOnComplete = $Get(pArgs(3))
+ Set quitOnError = $Get(pArgs(2))
+ Set haltOnComplete = $Get(pArgs(3))
Write !, "IPM is not enabled in this namespace."
&sql(SELECT %DLIST(Nsp) INTO :namespaces FROM %SYS.Namespace_List())
If SQLCODE '= 0 {
@@ -2722,16 +2722,16 @@ ZPM(pArgs...)
}
}
- If tQuitOnError {
+ If quitOnError {
Write ! // Add a newline before quitting, otherwise the shell prompt will be on the same line as the error message
Do $System.Process.Terminate($Job, 1)
}
- If tHaltOnComplete {
+ If haltOnComplete {
Write ! // Add a newline before halting
Halt
}
// The $$$ERROR macro is not available for use in language extension
- Return $$Error^%apiOBJ(5001,"IPM is not enabled in this namespace.")
+ Return $System.Status.Error(5001, "IPM is not enabled in this namespace.")
}
}
From bf96419d7d6c0a2049f5c630bf1b7ff79445bf5a Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 09:15:02 -0400
Subject: [PATCH 122/182] refactor: use %IPM.Main:IPMInstalled to retrieve IPM
module ID
---
src/cls/IPM/Main.cls | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 3eaf0784..d530de6b 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -2739,7 +2739,7 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
{
New $Namespace
Set initNamespace = $Namespace
- If ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName,.ipmModuleId) {
+ If ..IPMInstalled(.ipmModuleId){
Set modDef = ##class(%IPM.Storage.Module).%OpenId(ipmModuleId,,.sc)
$$$ThrowOnError(sc)
Write !, "Version of IPM in current namespace: "
@@ -3093,9 +3093,9 @@ ClassMethod GetMapInfo(Output pIsMappedFrom, Output pIsMappedTo)
}
}
-ClassMethod IPMInstalled() As %Boolean [ CodeMode = expression ]
+ClassMethod IPMInstalled(ByRef ipmModuleId) As %Boolean [ CodeMode = expression ]
{
-$$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName)
+$$$comClassDefined("%IPM.Storage.Module") && ##class(%IPM.Storage.Module).NameExists($$$IPMModuleName, .ipmModuleId)
}
/// Runs package manager commands in a way that is friendly to the OS-level shell.
From bad7332b51f288965c049a42e9193ebdcc3a62c2 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 10:14:08 -0400
Subject: [PATCH 123/182] enhance: remvoe redundant call to MapIfLegacy in
IPM.Installer.cls
---
preload/cls/IPM/Installer.cls | 1 -
1 file changed, 1 deletion(-)
diff --git a/preload/cls/IPM/Installer.cls b/preload/cls/IPM/Installer.cls
index 17d27074..4c67a5ca 100644
--- a/preload/cls/IPM/Installer.cls
+++ b/preload/cls/IPM/Installer.cls
@@ -35,7 +35,6 @@ XData PM [ XMLNamespace = INSTALLER ]
-
From d8ac1af6ff9eb81fe33184d9f5ae71271f0aa955 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 11:17:14 -0400
Subject: [PATCH 124/182] style: more minor changes
---
src/cls/IPM/Main.cls | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index d530de6b..6a3cbfa1 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -636,9 +636,9 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
Unmap %IPM package and routines in specified namespaces. Will Skip non-mapped namespaces.
-
+
@@ -2792,7 +2792,8 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
}
Set pointer = 0
While $ListNext(namespaces,pointer,namespace) {
- Set $Namespace = $Zstrip(namespace, "<>WC")
+ Set namespace = $Zstrip(namespace, "<>WC")
+ Set $Namespace = namespace
If ..IPMInstalled() {
If 'quiet || preview {
Write !,"Skipping "_namespace_" - IPM already installed."
From f7bb629acd5e5c0bcaed0155a8886098147cea15 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 11:17:44 -0400
Subject: [PATCH 125/182] feat: use dynamic SQL to prevent unexpected problems
---
src/cls/IPM/Main.cls | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 6a3cbfa1..dd494d25 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -2700,15 +2700,14 @@ ZPM(pArgs...)
Set quitOnError = $Get(pArgs(2))
Set haltOnComplete = $Get(pArgs(3))
Write !, "IPM is not enabled in this namespace."
- &sql(SELECT %DLIST(Nsp) INTO :namespaces FROM %SYS.Namespace_List())
- If SQLCODE '= 0 {
- Write !, "Error getting namespace name. SQLCODE =", SQLCODE
+ Set rs = ##class(%SQL.Statement).%ExecDirect(, "SELECT Nsp FROM %SYS.Namespace_List()")
+ If rs.%SQLCODE '= 0 {
+ Write !, "Error getting namespace name. SQLCODE =", rs.%SQLCODE
} Else {
New $Namespace
- Set ptr = 0
Set found = 0
- While $ListNext(namespaces, ptr, ns) {
- Set $Namespace = $Zstrip(ns, "<>WC")
+ While rs.%Next() {
+ Set $Namespace = $Zstrip(rs.%Get("Nsp"), "<>WC")
If $System.CLS.IsMthd("%IPM.Main", "Shell") {
Write !, "Change namepace to one of the following to run the ""zpm"" command"
Do ##class(%IPM.Main).Shell("version")
From 327eb175f7c4b2666aaf5679b432c2c82e2d75ed Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 7 Oct 2024 12:01:22 -0400
Subject: [PATCH 126/182] feat(enable): support -preview for all use cases
---
src/cls/IPM/Main.cls | 25 ++++++++++++++++++++-----
1 file changed, 20 insertions(+), 5 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index dd494d25..c229f17d 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -616,7 +616,7 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
-
+
enable -map -globally
@@ -2995,6 +2995,10 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
Quit
}
Set $Namespace = currentNS
+ If preview {
+ Write !, $$$FormatText("Would install IPM version %1 from registry in namespace %2", targetVersion, currentNS)
+ Continue
+ }
Do $SYSTEM.OBJ.LoadStream(manifest, loadFlag)
Write !, "IPM enabled for namespace "_currentNS
}
@@ -3009,6 +3013,11 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
Set tOverallSC = $$$OK
While (currentNS '="") {
Set $Namespace = currentNS
+ If preview {
+ Set filePath = ##class(%File).NormalizeFilename(targetIPMFileName, XMLDir)
+ Write !, $$$FormatText("Would install IPM using file at %1 in namespace %2", filePath, currentNS)
+ Continue
+ }
Set tSC = $SYSTEM.OBJ.Load(##class(%File).NormalizeFilename(targetIPMFileName, XMLDir), loadFlag)
If $$$ISOK(tSC) {
Set enabledNSList = enabledNSList_$ListBuild(currentNS)
@@ -3018,12 +3027,18 @@ ClassMethod EnableIPM(ByRef pCommandInfo)
Set currentNS = $ORDER(targetNamespaces(currentNS))
Set tOverallSC = $$$ADDSC(tOverallSC,tSC)
}
- Write !, "IPM enabled for namespace(s) "_$ListToString(enabledNSList)
- If $$$ISERR(tOverallSC) {
- Write !, "IPM failed to enable for namespace "_$ListToString(problematicList)
- $$$ThrowOnError(tOverallSC)
+ If 'preview {
+ Write !, "IPM enabled for namespace(s) "_$ListToString(enabledNSList)
+ If $$$ISERR(tOverallSC) {
+ Write !, "IPM failed to enable for namespace "_$ListToString(problematicList)
+ $$$ThrowOnError(tOverallSC)
+ }
}
}
+
+ If preview {
+ Write !,"Preview mode; no configuration changes were made."
+ }
}
ClassMethod UnmapIPM(ByRef pCommandInfo)
From 924306ddb2913e09cd10ce8296b1f2399141bc1c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 22 Oct 2024 14:51:08 -0400
Subject: [PATCH 127/182] fix: move namespace filtering from SQL to
ObjectScript for IRIS compat.
---
src/cls/IPM/Main.cls | 74 +++++++++++++++++++++++++++++++++++++++-----
1 file changed, 67 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index dd3a422a..b5315832 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -889,20 +889,40 @@ ClassMethod Namespace(ByRef pCommandInfo) [ Internal ]
/// Get list Namespace, example do ##class(%IPM.Main).GetListNamespace(.ns)
ClassMethod GetListNamespace(Output list, pSearch As %String = "")
{
+ Kill list
Set list = 0
+
+ // Build pattern for matching outside of SQL.
+ // Directly using `where Nsp LIKE ?` causes a bug described in https://github.com/intersystems/ipm/issues/579
+ Set pSearch = $Zstrip(pSearch, "<>WC")
+ If pSearch = "" {
+ Set pSearch = "*"
+ }
+ Set pieces = $ListFromString(pSearch, "*")
+ Set transformedPieces = ""
+ Set ptr = 0
+ While $ListNext(pieces, ptr, val) {
+ If val '? .(1AN,1"%") { // namespace
+ Return
+ }
+ Set transformedPieces = transformedPieces _ $ListBuild(1_$$$QUOTE(val))
+ }
+ Set pattern = $ListToString(transformedPieces, ".(1AN,1""%"")")
+ If pattern = "" { // should never happen
+ Set pattern = "0"""""
+ }
+
Set width = 0
Set tArgs = 0
Set tQuery = "SELECT Nsp FROM %SYS.Namespace_List()"
- If pSearch'="" {
- Set tQuery = tQuery _ " WHERE Nsp " _ $Select(pSearch["*": "LIKE", 1: "=") _ " ?"
- Set tArgs($Increment(tArgs)) = $Translate($$$UPPER(pSearch), "*", "%")
- }
Set tRes = ##class(%SQL.Statement).%ExecDirect(,tQuery, tArgs...)
$$$ThrowSQLIfError(tRes.%SQLCODE, tRes.%Message)
While tRes.%Next(.tSC) {
$$$ThrowOnError(tSC)
Set nsp = tRes.Nsp
- Set list(nsp) = ""
+ If nsp ? @pattern {
+ Set list(nsp) = ""
+ }
}
}
@@ -2221,9 +2241,49 @@ ClassMethod RunDependencyAnalyzer(ByRef pCommandInfo)
Write !
}
-Query ActiveNamespaces() As %SQLQuery(ROWSPEC = "ID:%String,Name:%String") [ SqlProc ]
+/// Implemented as custom query instead of `select Nsp, Nsp from %SYS.Namespace_List(0,0) WHERE status = 1` because of a DP issue, see https://github.com/intersystems/ipm/issues/579
+Query ActiveNamespaces() As %Query(ROWSPEC = "ID:%String,Name:%String") [ SqlProc ]
+{
+}
+
+ClassMethod ActiveNamespacesExecute(qHandle As %Binary) As %Status
+{
+ Try {
+ Set tQuery = "SELECT NSP, Status FROM %SYS.Namespace_List(0,0)"
+ Set rs = ##class(%SQL.Statement).%ExecDirect(, tQuery)
+ Kill qHandle
+ While rs.%Next() {
+ If rs.%Get("Status") {
+ Set qHandle($INCREMENT(qHandle)) = rs.%Get("NSP")
+ }
+ }
+ Set qHandle = 0
+ } catch ex {
+ Return ex.AsStatus()
+ }
+ Return $$$OK
+}
+
+ClassMethod ActiveNamespacesFetch(qHandle As %Binary, ByRef Row As %List, ByRef AtEnd As %Integer = 0) As %Status [ PlaceAfter = ActiveNamespacesExecute ]
+{
+ Try {
+ If $Data(qHandle($Increment(qHandle)), nsp) # 2 {
+ Set Row = $ListBuild(nsp, nsp)
+ Set AtEnd = 0
+ } Else {
+ Set Row = ""
+ Set AtEnd = 1
+ }
+ } catch ex {
+ Return ex.AsStatus()
+ }
+ Return $$$OK
+}
+
+ClassMethod ActiveNamespacesClose(qHandle As %Binary) As %Status [ PlaceAfter = ActiveNamespacesFetch ]
{
- select Nsp,Nsp from %SYS.Namespace_List(0,0) where Status = 1
+ Kill qHandle
+ Return $$$OK
}
ClassMethod ListInstalled(ByRef pCommandInfo) [ Private ]
From e53ba9eb6a5bc284f4920ec616862351c8909264 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 22 Oct 2024 14:55:42 -0400
Subject: [PATCH 128/182] style: minor comment and remove unused args
---
src/cls/IPM/Main.cls | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index b5315832..a991a912 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -902,7 +902,7 @@ ClassMethod GetListNamespace(Output list, pSearch As %String = "")
Set transformedPieces = ""
Set ptr = 0
While $ListNext(pieces, ptr, val) {
- If val '? .(1AN,1"%") { // namespace
+ If val '? .(1AN,1"%") { // namespace contain only numbers, letters and %, otherwise there's guaranteed no match
Return
}
Set transformedPieces = transformedPieces _ $ListBuild(1_$$$QUOTE(val))
@@ -913,9 +913,8 @@ ClassMethod GetListNamespace(Output list, pSearch As %String = "")
}
Set width = 0
- Set tArgs = 0
Set tQuery = "SELECT Nsp FROM %SYS.Namespace_List()"
- Set tRes = ##class(%SQL.Statement).%ExecDirect(,tQuery, tArgs...)
+ Set tRes = ##class(%SQL.Statement).%ExecDirect(,tQuery)
$$$ThrowSQLIfError(tRes.%SQLCODE, tRes.%Message)
While tRes.%Next(.tSC) {
$$$ThrowOnError(tSC)
From b62b2cda4267618ba61f25a075e4ed8a234a0789 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 23 Oct 2024 14:24:47 -0400
Subject: [PATCH 129/182] refactor: use regex instead of pattern matching to
check namespace value
---
src/cls/IPM/DataType/RegExString.cls | 15 +++++++++++++++
src/cls/IPM/Main.cls | 18 +++---------------
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/src/cls/IPM/DataType/RegExString.cls b/src/cls/IPM/DataType/RegExString.cls
index 85acbc00..27545819 100644
--- a/src/cls/IPM/DataType/RegExString.cls
+++ b/src/cls/IPM/DataType/RegExString.cls
@@ -50,4 +50,19 @@ ClassMethod IsValid(%val As %CacheString) As %Status [ ServerOnly = 0 ]
return $$$OK
}
+ClassMethod FromWildCard(wildcard As %String) As %String
+{
+ Set regex = ""
+ For i=1:1:$Length(wildcard) {
+ Set char = $Extract(wildcard, i)
+ If char = "*" {
+ Set regex = regex_".*"
+ } Else {
+ Set regex = regex_char
+ }
+ }
+ // Is there a way to return an instance of this class instead of %String?
+ Quit "^(?i)"_regex_"$"
+}
+
}
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index a991a912..a1a0a89b 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -892,25 +892,13 @@ ClassMethod GetListNamespace(Output list, pSearch As %String = "")
Kill list
Set list = 0
- // Build pattern for matching outside of SQL.
+ // Build regex for matching outside of SQL.
// Directly using `where Nsp LIKE ?` causes a bug described in https://github.com/intersystems/ipm/issues/579
Set pSearch = $Zstrip(pSearch, "<>WC")
If pSearch = "" {
Set pSearch = "*"
}
- Set pieces = $ListFromString(pSearch, "*")
- Set transformedPieces = ""
- Set ptr = 0
- While $ListNext(pieces, ptr, val) {
- If val '? .(1AN,1"%") { // namespace contain only numbers, letters and %, otherwise there's guaranteed no match
- Return
- }
- Set transformedPieces = transformedPieces _ $ListBuild(1_$$$QUOTE(val))
- }
- Set pattern = $ListToString(transformedPieces, ".(1AN,1""%"")")
- If pattern = "" { // should never happen
- Set pattern = "0"""""
- }
+ Set regex = ##class(%IPM.DataType.RegExString).FromWildCard(pSearch)
Set width = 0
Set tQuery = "SELECT Nsp FROM %SYS.Namespace_List()"
@@ -919,7 +907,7 @@ ClassMethod GetListNamespace(Output list, pSearch As %String = "")
While tRes.%Next(.tSC) {
$$$ThrowOnError(tSC)
Set nsp = tRes.Nsp
- If nsp ? @pattern {
+ If $Match(nsp, regex) {
Set list(nsp) = ""
}
}
From 37502995dd14f69bb8d24ef28c77142ba1882534 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 10:30:57 -0400
Subject: [PATCH 130/182] fix: make CSPApplication deprecated & forward
compatible
---
src/cls/IPM/Main.cls | 2 ++
src/cls/IPM/ResourceProcessor/CSPApplication.cls | 12 +++++++++---
src/inc/IPM/Common.inc | 5 ++++-
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index a1a0a89b..fa1a14f0 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -725,6 +725,7 @@ ClassMethod GetVersion(ModuleName, ByRef out, list)
/// For use in unit tests that need to test if a command threw any exceptions.
ClassMethod ShellInternal(pCommand As %String, Output pException As %Exception.AbstractException) [ Internal ]
{
+ New $$$DeprecationWarned
Set pException = $$$NULLOREF
Set tOneCommand = 0
Set tCommand = $Get(pCommand)
@@ -739,6 +740,7 @@ ClassMethod ShellInternal(pCommand As %String, Output pException As %Exception.A
)
Set tInShell = 0
For {
+ Kill $$$DeprecationWarned
Try {
// Have intro message just for first entrance to shell
diff --git a/src/cls/IPM/ResourceProcessor/CSPApplication.cls b/src/cls/IPM/ResourceProcessor/CSPApplication.cls
index 2af7190b..ba143c55 100644
--- a/src/cls/IPM/ResourceProcessor/CSPApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/CSPApplication.cls
@@ -1,4 +1,4 @@
-Include (%sySecurity, %occErrors)
+Include (%sySecurity, %occErrors, %IPM.Common, %IPM.Formatting)
Class %IPM.ResourceProcessor.CSPApplication Extends (%IPM.ResourceProcessor.Abstract, %XML.Adaptor, %Installer.CSPApplication) [ PropertyClass = %IPM.ResourceProcessor.PropertyParameters ]
{
@@ -31,10 +31,10 @@ Parameter XMLTYPE = "IPMCSPApplication";
Property Enabled As %Boolean [ InitialExpression = 1 ];
/// DeepSee Enabled
-Property DeepSeeEnabled As %Boolean [ InitialExpression = 0 ];
+Property DeepSeeEnabled As %Installer.Boolean [ InitialExpression = 0 ];
/// iKnow Enabled
-Property iKnowEnabled As %Boolean [ InitialExpression = 0 ];
+Property iKnowEnabled As %Installer.Boolean [ InitialExpression = 0 ];
/// Password authentication enabled
Property PasswordAuthEnabled As %Boolean [ InitialExpression = 0 ];
@@ -68,6 +68,12 @@ Property PermittedClasses As %String(MAXLEN = 32767);
Method %OnNew(pResourceReference As %IPM.Storage.ResourceReference) As %Status [ Private, ServerOnly = 1 ]
{
+ Set packageName = pResourceReference.Module.Name
+ If ($Get(packageName) '= "") && '$Data($$$DeprecationWarned(packageName)) {
+ Set $$$DeprecationWarned(packageName) = 1
+ Write !, $$$FormattedLine($$$Red, "WARNING: The resource tag is deprecated and may be removed in a future release of IPM.")
+ Write !, $$$FormattedLine($$$Red, $$$FormatText(" Please contact the package developer of %1 to use instead", packageName))
+ }
Set tSC = ##super(pResourceReference)
Set ..Url = $ZConvert(..Url,"L")
Quit tSC
diff --git a/src/inc/IPM/Common.inc b/src/inc/IPM/Common.inc
index 0d0b318d..97dd17d8 100644
--- a/src/inc/IPM/Common.inc
+++ b/src/inc/IPM/Common.inc
@@ -63,4 +63,7 @@ ROUTINE %IPM.Common [Type=INC]
#def1arg STARTTAGREGEX "^ ;Generated by (%IPM.Main|%ZPM.PackageManager): Start$"
#def1arg ENDTAGREGEX "^ ;Generated by (%IPM.Main|%ZPM.PackageManager): End$"
#def1arg STARTTAG ##Expression($$$STARTTAGQ)
-#def1arg ENDTAG ##Expression($$$ENDTAGQ)
\ No newline at end of file
+#def1arg ENDTAG ##Expression($$$ENDTAGQ)
+
+#; Variable to mark deprecation warning has been shown in the current process
+#define DeprecationWarned %IPMModuleDeprecatedResource
\ No newline at end of file
From e1ad78dde9d0104789fd6559ace9e35d1b091106 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 10:51:50 -0400
Subject: [PATCH 131/182] chore: add change log entry for webapplication
deprecation
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 865b12a8..05913bfb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,4 +64,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
-
### Deprecated
--
+- #593 CSPApplication is deprecated in favor of WebApplication. User will be warned when installing a package containing CSPApplication.
From 451a69c566c96a7a8e443ac706733a71cb351a4f Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 11:23:08 -0400
Subject: [PATCH 132/182] feat: allow bypass installation of python
dependencies
---
CHANGELOG.md | 1 +
src/cls/IPM/Lifecycle/Base.cls | 6 ++++++
src/cls/IPM/Main.cls | 9 +++++++++
3 files changed, 16 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 865b12a8..bd332b16 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
- #562 Added a generic resource processpor `WebApplication`, which handles creating and removal of all Security.Applications resources
- #575 Added ability to expand `$$$macro` in module.xml. The macro cannot take any arguments yet.
+- #595 Added ability to bypass installation of python dependencies with -DBypassPyDeps=1.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index a71247a2..a2068015 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -611,6 +611,12 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Quit $$$OK
}
Set tVerbose = $Get(pParams("Verbose"))
+ If $Get(pParams("BypassPyDeps"), 0) {
+ If tVerbose {
+ Write !, "Skipping installation of Python dependencies because BypassPyDeps is set."
+ }
+ Quit $$$OK
+ }
Set pythonRequirements = ##class(%File).NormalizeFilename("requirements.txt", pRoot)
If '##class(%File).Exists(pythonRequirements) {
Quit $$$OK
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index a1a0a89b..5104ec36 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -250,6 +250,9 @@ load -dev -verbose C:\module\root\path\module-0.0.1.tgz
load https://github.com/user/repository.git
load https://github.com/user/repository.git -branch feature-1
+
+ load -DBypassPyDeps=1 C:\module\root\path\
+
@@ -356,6 +359,9 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
install HS.JSON 1.x
+
+ install -DBypassPyDeps=1 HS.JSON
+
@@ -382,6 +388,9 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
reinstall -dev ZHSLIB
+
+ reinstall -DBypassPyDeps=1 ZHSLIB
+
From 5a9483e29b03ca5b00b743a691324b6aeab9f9e1 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 12:27:36 -0400
Subject: [PATCH 133/182] refactor: rename constant macros
---
src/inc/IPM/Common.inc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/inc/IPM/Common.inc b/src/inc/IPM/Common.inc
index 0d0b318d..5f408008 100644
--- a/src/inc/IPM/Common.inc
+++ b/src/inc/IPM/Common.inc
@@ -27,11 +27,11 @@ ROUTINE %IPM.Common [Type=INC]
#define ZPMDependencyTempDataNext $Increment($$$ZPMDependencyTempData)
/// Global to keep track of packages in the current namespace for invalid reference
/// checks in the dependency analyzer
-#define ZPMDependencyNsPackages ^IRIS.Temp.ZPMDepNsPackages
+#define ZPMDependencyNsPackages ^IRIS.Temp.IPMDepNsPackages
#define ZPMDefaultModifiers ^%IPM.DefaultModifiers
#; Temp global used to store source control output from CSP pages to be shown in Studio/Atelier console
-#define ZPMExtensionOutput ^IRIS.Temp.ZPMExtensionOutput($Username,$Namespace)
+#define ZPMExtensionOutput ^IRIS.Temp.IPMExtensionOutput($Username,$Namespace)
#define ZPMExtensionOutputClear Kill $$$ZPMExtensionOutput
#define ZPMExtensionOutputSet(%array) $$$ZPMExtensionOutputClear Merge $$$ZPMExtensionOutput = %array
#define ZPMExtensionOutputGet(%array) Merge %array = $$$ZPMExtensionOutput
From 0f29ba8cd4bb76d0b596224ba4d40fcd562b633c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 13:23:32 -0400
Subject: [PATCH 134/182] enhance: make bypass-py-deps a distinct modifier
---
src/cls/IPM/Lifecycle/Base.cls | 2 +-
src/cls/IPM/Main.cls | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index a2068015..08148163 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -613,7 +613,7 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Set tVerbose = $Get(pParams("Verbose"))
If $Get(pParams("BypassPyDeps"), 0) {
If tVerbose {
- Write !, "Skipping installation of Python dependencies because BypassPyDeps is set."
+ Write !, "Skipping installation of Python dependencies because -bypass-py-deps is set."
}
Quit $$$OK
}
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 5104ec36..288e0e8e 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -251,7 +251,7 @@ load https://github.com/user/repository.git
load https://github.com/user/repository.git -branch feature-1
- load -DBypassPyDeps=1 C:\module\root\path\
+ load -bypass-py-deps C:\module\root\path\
@@ -261,6 +261,7 @@ load https://github.com/user/repository.git -branch feature-1
+
@@ -360,7 +361,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
install HS.JSON 1.x
- install -DBypassPyDeps=1 HS.JSON
+ install -bypass-py-deps HS.JSON
@@ -374,6 +375,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
+
@@ -389,7 +391,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
reinstall -dev ZHSLIB
- reinstall -DBypassPyDeps=1 ZHSLIB
+ reinstall -bypass-py-deps ZHSLIB
@@ -400,6 +402,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
+
From 438eb4e2dfd2d4e37e4598cafa4c38c85f4dae6e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 13:24:44 -0400
Subject: [PATCH 135/182] docs: update changelog
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bd332b16..97dbdcb5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #538 Added ability to customize caller to PipCaller and UseStandalonePip through `config set`, which are empty by default and can be used to override the auto-detection of pip.
- #562 Added a generic resource processpor `WebApplication`, which handles creating and removal of all Security.Applications resources
- #575 Added ability to expand `$$$macro` in module.xml. The macro cannot take any arguments yet.
-- #595 Added ability to bypass installation of python dependencies with -DBypassPyDeps=1.
+- #595 Added ability to bypass installation of python dependencies with -bypass-py-deps or -DBypassPyDeps=1.
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
From fd635248e9bad948793126caea2ed79de026fe2b Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 24 Oct 2024 15:24:51 -0400
Subject: [PATCH 136/182] fix: SQL failures due to typo in semver Postrelease
SQL compute code
---
src/cls/IPM/General/SemanticVersion.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/General/SemanticVersion.cls b/src/cls/IPM/General/SemanticVersion.cls
index 1c32886a..19e13282 100644
--- a/src/cls/IPM/General/SemanticVersion.cls
+++ b/src/cls/IPM/General/SemanticVersion.cls
@@ -14,7 +14,7 @@ Property Patch As %Integer(MINVAL = 0) [ Required ];
Property Prerelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
/// This is an alias for Prerelease. It is used for code readability when SemVerPostRelease is enabled.
-Property Postrelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*") [ Calculated, SqlComputeCode = { Set {*} = Prerelease }, SqlComputed, Transient ];
+Property Postrelease As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*") [ Calculated, SqlComputeCode = { Set {*} = {Prerelease} }, SqlComputed, Transient ];
Property Build As %IPM.DataType.RegExString(MAXLEN = 100, REGEX = "([0-9A-Za-z-])+(\.([0-9A-Za-z-])+)*");
From 1ef84094fd9b874f95088a0955c24bff25b86960 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 25 Oct 2024 10:56:15 -0400
Subject: [PATCH 137/182] feat: warn that dbrole is deprecated when evaluating
it
---
src/cls/IPM/ResourceProcessor/CSPApplication.cls | 4 ++--
src/cls/IPM/Utils/Module.cls | 6 +++++-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/CSPApplication.cls b/src/cls/IPM/ResourceProcessor/CSPApplication.cls
index ba143c55..80b79d2c 100644
--- a/src/cls/IPM/ResourceProcessor/CSPApplication.cls
+++ b/src/cls/IPM/ResourceProcessor/CSPApplication.cls
@@ -69,8 +69,8 @@ Property PermittedClasses As %String(MAXLEN = 32767);
Method %OnNew(pResourceReference As %IPM.Storage.ResourceReference) As %Status [ Private, ServerOnly = 1 ]
{
Set packageName = pResourceReference.Module.Name
- If ($Get(packageName) '= "") && '$Data($$$DeprecationWarned(packageName)) {
- Set $$$DeprecationWarned(packageName) = 1
+ If ($Get(packageName) '= "") && '$Data($$$DeprecationWarned(packageName, "CSPApplication")) {
+ Set $$$DeprecationWarned(packageName, "CSPApplication") = 1
Write !, $$$FormattedLine($$$Red, "WARNING: The resource tag is deprecated and may be removed in a future release of IPM.")
Write !, $$$FormattedLine($$$Red, $$$FormatText(" Please contact the package developer of %1 to use instead", packageName))
}
diff --git a/src/cls/IPM/Utils/Module.cls b/src/cls/IPM/Utils/Module.cls
index 5cec60ba..c6ca19d7 100644
--- a/src/cls/IPM/Utils/Module.cls
+++ b/src/cls/IPM/Utils/Module.cls
@@ -1,4 +1,4 @@
-Include (%occInclude, %occErrors, %syConfig, %syPrompt, %IPM.Common)
+Include (%occInclude, %occErrors, %syConfig, %syPrompt, %IPM.Common, %IPM.Formatting)
Class %IPM.Utils.Module [ System = 3 ]
{
@@ -2288,6 +2288,10 @@ ClassMethod %EvaluateSystemExpression(pString As %String) As %String [ Internal
Do ##class(%Studio.General).GetWebServerPort(,,,.urlRoot)
Set result = ..%RegExReplace(result, "webroot", urlRoot)
+ If '$Get($$$DeprecationWarned("", "dbrole")) && ((result [ "${dbrole}") || (result [ "{$dbrole}")) {
+ Set $$$DeprecationWarned("", "dbrole") = 1
+ Write !, $$$FormattedLine($$$Red, "WARNING: The {$dbrole}/${dbrole} expression is deprecated. Please contact package developer to use {$globalsDbRole} instead.")
+ }
Set dbRole = ..GetDatabaseRole()
Set result = ..%RegExReplace(result, "dbrole", dbRole)
Set result = ..%RegExReplace(result, "globalsDbRole", dbRole)
From b50747914257b8ffbc0746f59597f541b06879dd Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Fri, 25 Oct 2024 14:29:15 -0400
Subject: [PATCH 138/182] fix(docker): OS file permission isn't always
preserved in container
---
scripts/setup-registry.sh | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/scripts/setup-registry.sh b/scripts/setup-registry.sh
index 516543b5..5797fd8c 100644
--- a/scripts/setup-registry.sh
+++ b/scripts/setup-registry.sh
@@ -1,3 +1,4 @@
-#!/bin/env iriscli
-
-zpm "install zpm-registry"
\ No newline at end of file
+/bin/env iriscli << EOF
+zpm "install zpm-registry"
+halt
+EOF
\ No newline at end of file
From 884bc574efa68c558a7cb65aa81413eccb0d5ac7 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 30 Oct 2024 13:36:27 -0400
Subject: [PATCH 139/182] fix: report deletion of things that aren't classes
Also ensures we upper-case extensions (!) so we properly catch all classes/packages
%Library.RoutineMgr:Delete doesn't support qualifiers
---
CHANGELOG.md | 1 +
src/cls/IPM/Lifecycle/Base.cls | 9 ++++++---
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c88a092e..83dd1048 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #544: When installing a package from remote repo, IPM specifies `includePrerelease` and `includeSnapshots` in HTTP request. Correctly-behaving zpm registry should respect that.
- #557: When comparing semver against semver expressions, exclude prereleases and snapshots from the range maximum.
- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
+- #607: Uninstall reports deletion of non-classes
### Security
-
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 08148163..60a60b41 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -281,9 +281,9 @@ Method %Clean(ByRef pParams) As %Status
}
Set tName = $Piece(tChildKey,".",1,*-1)
- Set tExt = $Piece(tChildKey,".",*)
- Continue:tExt=""
- Continue:tName=""
+ Set tExt = $ZConvert($Piece(tChildKey,".",*),"U")
+ Continue:tExt=""
+ Continue:tName=""
Set tDeleteArray(tExt,tName) = ""
}
}
@@ -344,6 +344,9 @@ Method %Clean(ByRef pParams) As %Status
Continue
}
+ If tVerbose {
+ Write !,"Deleting ",tFullName
+ }
Set tDelSC = ##class(%Library.RoutineMgr).Delete(tFullName)
If $$$ISERR(tDelSC) {
Write !,"WARNING: "_$System.Status.GetErrorText(tDelSC)
From c603f38548afc55b1f0cebd0b84fa07ca237b3de Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 30 Oct 2024 14:21:33 -0400
Subject: [PATCH 140/182] fix: tar archive has garbage folders
Fixes #606
---
CHANGELOG.md | 1 +
src/cls/IPM/Utils/FileBinaryTar.cls | 11 ++++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c88a092e..2b5fcd3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #544: When installing a package from remote repo, IPM specifies `includePrerelease` and `includeSnapshots` in HTTP request. Correctly-behaving zpm registry should respect that.
- #557: When comparing semver against semver expressions, exclude prereleases and snapshots from the range maximum.
- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
+- #606: Don't put garbage folders in tar archive
### Security
-
diff --git a/src/cls/IPM/Utils/FileBinaryTar.cls b/src/cls/IPM/Utils/FileBinaryTar.cls
index baaa9caa..6e29154c 100644
--- a/src/cls/IPM/Utils/FileBinaryTar.cls
+++ b/src/cls/IPM/Utils/FileBinaryTar.cls
@@ -321,6 +321,10 @@ ClassMethod ConstructTar(Path As %String, RelativeTo As %String, archive As %Str
set flag = 5 // Directory
set size = 0
set name = ##class(%File).GetFilename($Extract(Path,1,*-1))
+ set prefix = $Extract(prefix,1,*-$Length(name)-1)
+ if $$$isWINDOWS {
+ set prefix = $Replace(prefix,"\","/")
+ }
} else {
set Path = ##class(%File).NormalizeFilename(Path)
set prefix = ##class(%File).GetDirectory(Path)
@@ -328,6 +332,9 @@ ClassMethod ConstructTar(Path As %String, RelativeTo As %String, archive As %Str
set flag = 0 // Ordinary File
set size = ##class(%File).GetFileSize(Path)
set name = ##class(%File).GetFilename(Path)
+ if $$$isWINDOWS {
+ set prefix = $Replace(prefix,"\","/")
+ }
}
set mtime = ##class(%File).GetFileDateModified(Path, 1)
@@ -366,7 +373,9 @@ ClassMethod ConstructTar(Path As %String, RelativeTo As %String, archive As %Str
do archive.Write(padding)
}
} else {
- do archive.Write(header)
+ if (Path '= RelativeTo) {
+ do archive.Write(header)
+ }
set rs = ##class(%File).FileSetFunc(Path, , "Type", 1)
while rs.%Next() {
do ..ConstructTar(rs.Name, RelativeTo, archive)
From ff17c34843e4016e1fdbd04f3a3cc01212a55a65 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 30 Oct 2024 14:57:35 -0400
Subject: [PATCH 141/182] test: add integration test for FileBinaryTar
This makes sure we don't get the garbage directories and we do get the important things.
---
.../Test/PM/Integration/FileBinaryTar.cls | 41 +++++++++++++++++++
1 file changed, 41 insertions(+)
create mode 100644 tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
diff --git a/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls b/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
new file mode 100644
index 00000000..dc51dc4a
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
@@ -0,0 +1,41 @@
+/// Most convenient to represent this as an integration test
+/// to reuse the integration test packages.
+Class Test.PM.Integration.FileBinaryTar Extends %UnitTest.TestCase
+{
+
+Method TestPackageAndExtract()
+{
+ Set tSC = $$$OK
+ Try {
+ Set tTestRoot = ##class(%File).NormalizeDirectory($Get(^UnitTestRoot))
+
+ Set tModuleDir = ##class(%File).NormalizeDirectory(##class(%File).GetDirectory(tTestRoot)_"/_data/simple-module/")
+ Set tSC = ##class(%IPM.Main).Shell("load "_tModuleDir)
+ Do $$$AssertStatusOK(tSC,"Loaded SimpleModule module successfully.")
+
+ Set tempDir = ##class(%Library.File).TempFilename()_"dir"
+ Set tSC = ##class(%IPM.Main).Shell("simplemodule package -only -DPath="_tempDir)
+ Do $$$AssertStatusOK(tSC,"Packaged SimpleModule successfully.")
+
+ Set outFile = ##class(%Library.File).TempFilename()
+ Set outDir = ##class(%Library.File).NormalizeDirectory(##class(%Library.File).TempFilename()_"dir-out")
+ Do ##class(%Library.File).CreateDirectoryChain(outDir)
+ Do $$$AssertEquals($zf(-100,"/STDOUT="_outFile_"/STDERR="_outFile,"tar","-xvf",tempDir_".tgz","-C",outDir),0)
+
+ Do $$$AssertNotTrue(##class(%File).DirectoryExists(outDir_"src/cls/Test/Test"))
+ Do $$$AssertNotTrue(##class(%File).DirectoryExists(outDir_"src/src"))
+ Do $$$AssertNotTrue(##class(%File).DirectoryExists(outDir_"simplemodule"))
+ Do $$$AssertTrue(##class(%File).Exists(outDir_"src/cls/Test/Test.cls"))
+ Do $$$AssertTrue(##class(%File).Exists(outDir_"module.xml"))
+
+ Set tSC = ##class(%IPM.Main).Shell("load "_outFile)
+ Do $$$AssertStatusOK(tSC,"Loaded SimpleModule module successfully from .tgz file.")
+
+ Set tSC = ##class(%IPM.Main).Shell("load "_outDir)
+ Do $$$AssertStatusOK(tSC,"Loaded SimpleModule module successfully from package directory.")
+ } Catch e {
+ Do $$$AssertStatusOK(e.AsStatus(),"An exception occurred.")
+ }
+}
+
+}
From 3b8d828b114df1b933353e2f5a7c0cc70b3b9750 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Mon, 4 Nov 2024 08:49:32 -0500
Subject: [PATCH 142/182] test: quote files
Of course this failed on Linux because Linux slashes look like other invalid arguments. Totally obvious after you dig into $zu(56,2)...
---
tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls b/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
index dc51dc4a..22d4e4c7 100644
--- a/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
+++ b/tests/integration_tests/Test/PM/Integration/FileBinaryTar.cls
@@ -20,7 +20,7 @@ Method TestPackageAndExtract()
Set outFile = ##class(%Library.File).TempFilename()
Set outDir = ##class(%Library.File).NormalizeDirectory(##class(%Library.File).TempFilename()_"dir-out")
Do ##class(%Library.File).CreateDirectoryChain(outDir)
- Do $$$AssertEquals($zf(-100,"/STDOUT="_outFile_"/STDERR="_outFile,"tar","-xvf",tempDir_".tgz","-C",outDir),0)
+ Do $$$AssertEquals($zf(-100,"/STDOUT="""_outFile_"""/STDERR="""_outFile_"""","tar","-xvf",tempDir_".tgz","-C",outDir),0)
Do $$$AssertNotTrue(##class(%File).DirectoryExists(outDir_"src/cls/Test/Test"))
Do $$$AssertNotTrue(##class(%File).DirectoryExists(outDir_"src/src"))
From c6d9c080e14593f38ffe28d1d039c789ecde5d89 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Tue, 12 Nov 2024 08:46:11 -0500
Subject: [PATCH 143/182] chore: comment on slashes
---
src/cls/IPM/Utils/FileBinaryTar.cls | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/cls/IPM/Utils/FileBinaryTar.cls b/src/cls/IPM/Utils/FileBinaryTar.cls
index 6e29154c..c0e622f8 100644
--- a/src/cls/IPM/Utils/FileBinaryTar.cls
+++ b/src/cls/IPM/Utils/FileBinaryTar.cls
@@ -323,6 +323,7 @@ ClassMethod ConstructTar(Path As %String, RelativeTo As %String, archive As %Str
set name = ##class(%File).GetFilename($Extract(Path,1,*-1))
set prefix = $Extract(prefix,1,*-$Length(name)-1)
if $$$isWINDOWS {
+ // Archive should have Unix-style slashes, so change to that on Windows
set prefix = $Replace(prefix,"\","/")
}
} else {
@@ -333,6 +334,7 @@ ClassMethod ConstructTar(Path As %String, RelativeTo As %String, archive As %Str
set size = ##class(%File).GetFileSize(Path)
set name = ##class(%File).GetFilename(Path)
if $$$isWINDOWS {
+ // Archive should have Unix-style slashes, so change to that on Windows
set prefix = $Replace(prefix,"\","/")
}
}
From 5b8381aa2a8492ab1377f1e541a29c649a94edfd Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 11 Nov 2024 09:17:39 -0500
Subject: [PATCH 144/182] fix(ci): exclude community containers with expired
licences
---
.github/workflows/images.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
index 097ce86d..d118a661 100644
--- a/.github/workflows/images.yml
+++ b/.github/workflows/images.yml
@@ -31,7 +31,7 @@ jobs:
do
# Skip irishealth-community due to bad interaction with ZPM document type
# Also skip 2023.2 because the license has expired
- if [ "$n" = "irishealth-community" -a "$tag" = "2023.3" -o "$tag" = "2023.2" ];
+ if [ "$tag" = "2023.3" -o "$tag" = "2023.2" ];
then
continue
fi
From d0564f54a5c472f12f158dafefaab196dabb6186 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Tue, 19 Nov 2024 15:41:30 -0500
Subject: [PATCH 145/182] $ListFind("CLS","MAC","XML") had "XML" removed from
it which lead to LOC files failing to import
---
src/cls/IPM/ResourceProcessor/Default/Document.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index e4f649d0..d2832077 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -421,7 +421,7 @@ Method OnLoad(pFullResourcePath As %String, pVerbose As %Boolean, pCompile As %B
} Else {
Set tProcessed = 0
set fileExt = $$$UPPER($Piece(pFullResourcePath, ".", *))
- if '$LISTFIND($LB("CLS", "MAC"), fileExt), ##class(%RoutineMgr).UserType(..ResourceReference.Name, .docclass, .doctype) {
+ if '$LISTFIND($LB("CLS", "MAC", "XML"), fileExt), ##class(%RoutineMgr).UserType(..ResourceReference.Name, .docclass, .doctype) {
try {
set stream = ##class(%Stream.FileCharacter).%New()
$$$ThrowOnError(stream.LinkToFile(pFullResourcePath))
From 8303e6cd651e12d20aa6b7afba09d447f21abad8 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 20 Nov 2024 07:45:45 -0500
Subject: [PATCH 146/182] chore(ci): update upload-artifact to v4
---
.github/workflows/packages.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 253e8aba..fd0a9d4d 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -59,7 +59,7 @@ jobs:
EOF
docker container stop $CONTAINER
- name: Upload Image
- uses: actions/upload-artifact@v2
+ uses: actions/upload-artifact@v4
with:
name: zpmimage
path: /tmp/zpmimage.tar
From adb3e65b1e999ef7966e7a1796ebfe9a3c7bbc77 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Wed, 20 Nov 2024 10:38:16 -0500
Subject: [PATCH 147/182] Updating artifact to 0.7.3 for CI checks
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 64332c5a..070b50d9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -169,7 +169,7 @@ jobs:
run: |
curl http://localhost:52773/registry/packages/-/all | jq
curl http://localhost:52773/registry/packages/zpm/ | jq
- ASSET_NAME='zpm-0.7.2.xml'
+ ASSET_NAME='zpm-0.7.3.xml'
ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
wget $ASSET_URL -O /tmp/zpm.xml
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
From 09ec9593682f36f018089ec5463b7e950809f48d Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 2 Dec 2024 12:19:38 -0500
Subject: [PATCH 148/182] feat(ci): add timeout for testing packages
---
.github/workflows/packages.yml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index fd0a9d4d..665ba829 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -114,9 +114,10 @@ jobs:
echo "::group::Test package $package"
set +e
- docker exec -i $CONTAINER iris session IRIS <<- EOF
- zpm "install $package":1
+ timeout 60s docker exec -i $CONTAINER iris session IRIS <<- EOF
+ zpm "install $package":1
zpm "$package test -only ${{ env.test-flags }}":1:1
+ halt
EOF
if [ $? -ne 0 ]; then
From 16d50c4d3d047be06e58be3bda02ca94f47cdc6e Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 2 Dec 2024 12:34:33 -0500
Subject: [PATCH 149/182] fix(ci): bump legacy zpm version to 0.7.3
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 64332c5a..070b50d9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -169,7 +169,7 @@ jobs:
run: |
curl http://localhost:52773/registry/packages/-/all | jq
curl http://localhost:52773/registry/packages/zpm/ | jq
- ASSET_NAME='zpm-0.7.2.xml'
+ ASSET_NAME='zpm-0.7.3.xml'
ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
wget $ASSET_URL -O /tmp/zpm.xml
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
From bdc38b97dc99c162a375293b24412b329f2e746a Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Mon, 2 Dec 2024 17:10:36 -0500
Subject: [PATCH 150/182] Changing from "text()" to "node()" instead of adding
"XML"
---
src/cls/IPM/ResourceProcessor/Default/Document.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index d2832077..27df2e5d 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -428,7 +428,7 @@ Method OnLoad(pFullResourcePath As %String, pVerbose As %Boolean, pCompile As %B
try {
Quit:'##class(%XML.XPATH.Document).CreateFromStream(stream, .tDocument)
Quit:'$ISOBJECT(tDocument)
- Quit:'tDocument.EvaluateExpression("/Export/Document", "text()", .tRes)
+ Quit:'tDocument.EvaluateExpression("/Export/Document", "node()", .tRes)
If $IsObject(tRes),tRes.Size {
Set tSC = $System.OBJ.Load(pFullResourcePath,tFlags,,.pLoadedList)
Set tProcessed = 1
From b05860b13f0622e22c78e10cfaad8af343e470d6 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Mon, 2 Dec 2024 17:14:20 -0500
Subject: [PATCH 151/182] Forgot to remove the "XML" part
---
src/cls/IPM/ResourceProcessor/Default/Document.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index 27df2e5d..8c160553 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -421,7 +421,7 @@ Method OnLoad(pFullResourcePath As %String, pVerbose As %Boolean, pCompile As %B
} Else {
Set tProcessed = 0
set fileExt = $$$UPPER($Piece(pFullResourcePath, ".", *))
- if '$LISTFIND($LB("CLS", "MAC", "XML"), fileExt), ##class(%RoutineMgr).UserType(..ResourceReference.Name, .docclass, .doctype) {
+ if '$LISTFIND($LB("CLS", "MAC"), fileExt), ##class(%RoutineMgr).UserType(..ResourceReference.Name, .docclass, .doctype) {
try {
set stream = ##class(%Stream.FileCharacter).%New()
$$$ThrowOnError(stream.LinkToFile(pFullResourcePath))
From 2d1c8a6497aec5e81fc1de7ee151c0cccdf4cba8 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Dec 2024 10:57:36 -0500
Subject: [PATCH 152/182] fix(ci): map IPM to %SYS during build test phase
---
.github/workflows/packages.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 665ba829..1c9f4c60 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -54,6 +54,7 @@ jobs:
docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker exec -i $CONTAINER iris session IRIS << EOF
zpm "list":1
+ zpm "enable -g -map":1
zn "%SYS"
zpm "test zpm -v -only":1:1
EOF
From 73481a88aa5a40e51463a463cb9b6187beb478a9 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Dec 2024 11:06:01 -0500
Subject: [PATCH 153/182] fix: remove namespace switch
---
.github/workflows/packages.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 1c9f4c60..506039be 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -54,8 +54,6 @@ jobs:
docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker exec -i $CONTAINER iris session IRIS << EOF
zpm "list":1
- zpm "enable -g -map":1
- zn "%SYS"
zpm "test zpm -v -only":1:1
EOF
docker container stop $CONTAINER
From a026459e6062fea47b2f77d1b47d0fc98af1ae8f Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 3 Dec 2024 11:17:01 -0500
Subject: [PATCH 154/182] fix(ci): bump download-artifact to v4
---
.github/workflows/packages.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 506039be..ad5a118f 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -77,7 +77,7 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Download Artifact
- uses: actions/download-artifact@v2
+ uses: actions/download-artifact@v4
with:
name: zpmimage
path: /tmp
From e7db117992c75a52691c8ab1a57f5fa2dab2d5c7 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 4 Dec 2024 10:40:51 -0500
Subject: [PATCH 155/182] feat(ci): allow parameterization of package test
timeout
---
.github/workflows/packages.yml | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index ad5a118f..6c3c1426 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -12,6 +12,11 @@ on:
required: true
type: number
default: 1
+ timeoutSeconds:
+ description: "Timeout in seconds for each test"
+ required: true
+ type: number
+ default: 120
jobs:
matrix-setup:
@@ -113,10 +118,10 @@ jobs:
echo "::group::Test package $package"
set +e
- timeout 60s docker exec -i $CONTAINER iris session IRIS <<- EOF
- zpm "install $package":1
- zpm "$package test -only ${{ env.test-flags }}":1:1
- halt
+ timeout ${{ inputs.timeoutSeconds }}s docker exec -i $CONTAINER iris session IRIS <<- EOF
+ zpm "install $package":1
+ zpm "$package test -only ${{ env.test-flags }}":1:1
+ halt
EOF
if [ $? -ne 0 ]; then
From c7b1b084698dccf92fa286aab52ffb6e6749c5eb Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Wed, 4 Dec 2024 15:50:58 -0500
Subject: [PATCH 156/182] Adding unit test for .LOC files
---
.../resource-test/localize/HTTP/Statuses.LOC | 73 +++++++++++++++++++
.../_data/resource-test/module.xml | 1 +
2 files changed, 74 insertions(+)
create mode 100644 tests/integration_tests/Test/PM/Integration/_data/resource-test/localize/HTTP/Statuses.LOC
diff --git a/tests/integration_tests/Test/PM/Integration/_data/resource-test/localize/HTTP/Statuses.LOC b/tests/integration_tests/Test/PM/Integration/_data/resource-test/localize/HTTP/Statuses.LOC
new file mode 100644
index 00000000..5171fd7c
--- /dev/null
+++ b/tests/integration_tests/Test/PM/Integration/_data/resource-test/localize/HTTP/Statuses.LOC
@@ -0,0 +1,73 @@
+
+
+
+
+
+ 100 Continue %1
+ 101 Switching Protocols %1
+ 102 Processing %1
+ 200 OK %1
+ 201 Created %1
+ 202 Accepted %1
+ 203 Non-Authoritative Information %1
+ 204 No Content %1
+ 205 Reset Content %1
+ 206 Partial Content %1
+ 207 Multi-Status %1
+ 208 Already Reported %1
+ 226 Instance Manipulation Used %1
+ 300 Multiple Choices %1
+ 301 Moved Permanently %1
+ 302 Found %1
+ 303 See Other %1
+ 304 Not Modified %1
+ 305 Use Proxy %1
+ 306 Switch Proxy %1
+ 307 Temporary Redirect %1
+ 308 Permanent Redirect %1
+ 400 Bad Request %1
+ 401 Unauthorized %1
+ 402 Payment Required %1
+ 403 Forbidden %1
+ 404 Not Found %1
+ 405 Method Not Allowed %1
+ 406 Not Acceptable %1
+ 407 Proxy Authentication Required %1
+ 408 Request Timeout %1
+ 409 Conflict %1
+ 410 Gone %1
+ 411 Length Required %1
+ 412 Precondition Failed %1
+ 413 Payload Too Large %1
+ 414 URI Too Long %1
+ 415 Unsupported Media Type %1
+ 416 Range Not Satisfiable %1
+ 417 Expectation Failed %1
+ 418 I'm a teapot %1
+ 421 Misdirected Request %1
+ 422 Unprocessable Entity %1
+ 423 Locked %1
+ 424 Failed Dependency %1
+ 426 Upgrade Required %1
+ 428 Precondition Required %1
+ 429 Too Many Requests %1
+
+ 440 Login Timeout %1
+ 449 Retry With %1
+ 451 Unavailable For Legal Reasons %1
+ 500 Internal Server Error %1
+ 501 Not Implemented %1
+ 502 Bad Gateway %1
+ 503 Service Unavailable %1
+ 504 Gateway Timeout %1
+ 505 HTTP Version Not Supported %1
+ 506 Variant Also Negotiates %1
+ 507 Insufficient Storage %1
+ 508 Loop Detected %1
+ 509 Bandwidth Limit Exceeded %1
+ 510 Not Extended %1
+ 511 Network Authentication Required %1
+ 598 Network Read Timeout Error %1
+ 599 Network Connect Timeout Error %1
+
+
diff --git a/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml b/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
index 59edbcef..9f8c034b 100644
--- a/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
+++ b/tests/integration_tests/Test/PM/Integration/_data/resource-test/module.xml
@@ -10,6 +10,7 @@
+
misc
From 570cff6fdcb7be912c796a4d6ea8e62a135b0d09 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Thu, 5 Dec 2024 08:10:55 -0500
Subject: [PATCH 157/182] chore(ci): make /home/irisowner/zpm writable
Hopefully will fix CI writing to /localize
---
.github/workflows/main.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 070b50d9..abee6190 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -97,6 +97,7 @@ jobs:
zpm ${{ steps.image.outputs.flags }})
sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker cp . $CONTAINER:/home/irisowner/zpm/
+ docker exec -it --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/
echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl`
docker exec -i $CONTAINER iris session iris -UUSER << EOF
zpm "list":1
From b89d0519e6e4fd22b36c27e16bae93af1b5a63b1 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Thu, 5 Dec 2024 08:23:38 -0500
Subject: [PATCH 158/182] chore(ci): tweak docker exec
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index abee6190..0c1df545 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -97,7 +97,7 @@ jobs:
zpm ${{ steps.image.outputs.flags }})
sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker cp . $CONTAINER:/home/irisowner/zpm/
- docker exec -it --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/
+ echo `docker exec -i --user root $CONTAINER chmod -R 777 /home/irisowner/zpm/`
echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl`
docker exec -i $CONTAINER iris session iris -UUSER << EOF
zpm "list":1
From 875a4192c732941092d705d2b5557b8073510b00 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Dec 2024 09:57:10 -0500
Subject: [PATCH 159/182] feat: allow custom extra flags for pip install
---
src/cls/IPM/Lifecycle/Base.cls | 3 ++-
src/cls/IPM/Main.cls | 14 +++++++++++++-
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 60a60b41..60d71170 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -620,6 +620,7 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
}
Quit $$$OK
}
+ Set tExtraPipFlags = $ZStrip($Get(pParams("ExtraPipFlags"), ""), "<>WC")
Set pythonRequirements = ##class(%File).NormalizeFilename("requirements.txt", pRoot)
If '##class(%File).Exists(pythonRequirements) {
Quit $$$OK
@@ -638,7 +639,7 @@ Method InstallPythonRequirements(pRoot As %String = "", ByRef pParams)
Set tPyMinor = tSysModule."version_info".minor
Set tPyMicro = tSysModule."version_info".micro
Set tPyVersion = tPyMajor_"."_tPyMinor_"."_tPyMicro
- Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target, "--python-version", tPyVersion, "--only-binary=:all:")
+ Set command = ..ResolvePipCaller(.pParams) _ $ListBuild("install", "-r", "requirements.txt", "-t", target, "--python-version", tPyVersion, "--only-binary=:all:") _ $ListFromString(tExtraPipFlags, " ")
If tVerbose {
Write !, "Running "
Zwrite command
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index be7e8a62..fd99c005 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -253,6 +253,9 @@ load https://github.com/user/repository.git -branch feature-1
load -bypass-py-deps C:\module\root\path\
+
+ load -extra-pip-flags "--timeout 30" C:\module\root\path\
+
@@ -262,6 +265,7 @@ load https://github.com/user/repository.git -branch feature-1
+
@@ -363,6 +367,9 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
install -bypass-py-deps HS.JSON
+
+ install -extra-pip-flags "--timeout 30" HS.JSON
+
@@ -376,6 +383,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
+
@@ -393,6 +401,9 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
reinstall -bypass-py-deps ZHSLIB
+
+ reinstall -extra-pip-flags "--timeout 30" ZHSLIB
+
@@ -403,6 +414,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
+
@@ -2041,7 +2053,7 @@ ClassMethod Install(ByRef pCommandInfo) [ Internal ]
If (tResult '= "") {
Do ##class(%IPM.Lifecycle.Base).GetDefaultParameters(.tParams)
Merge tParams = pCommandInfo("data")
- Set tParams("DeveloperMode") = $Get(tParams("DeveloperMode"), 0)
+ Set tParams("DeveloperMode") = $Get(tParams("DeveloperMode"), 0)
Set tParams("cmd") = "install"
Set tParams("Install") = 1
If tResult.Deployed {
From d1fdfa4532392288919928cd8010b498326b4549 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Dec 2024 12:35:41 -0500
Subject: [PATCH 160/182] feat: auto update legacy abstract resource processor
before compiling
---
src/cls/IPM/Utils/LegacyCompat.cls | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
diff --git a/src/cls/IPM/Utils/LegacyCompat.cls b/src/cls/IPM/Utils/LegacyCompat.cls
index 2f1b473f..98ad797b 100644
--- a/src/cls/IPM/Utils/LegacyCompat.cls
+++ b/src/cls/IPM/Utils/LegacyCompat.cls
@@ -16,13 +16,19 @@ ClassMethod UpdateSuperclassAndCompile(ByRef pItems, ByRef qSpec = "") As %Statu
Continue
}
Set tClass = $EXTRACT(tName, 1, *-4)
- // Assuming this is the only one we care about. Eventually it may become a list.
- Set tOldLifecycle = "%ZPM.PackageManager.Developer.Lifecycle.Module"
- Set tNewLifecycle = "%IPM.Lifecycle.Module"
- If $$$defClassKeyGet(tClass, $$$cCLASSsuper) = tOldLifecycle {
- $$$defClassKeySet(tClass, $$$cCLASSsuper, tNewLifecycle)
- Write !, $$$FormattedLine($$$Magenta, "WARNING: ")
- Write tName _ " extends the deprecated class " _ tOldLifecycle _ ". It has been updated to " _ tNewLifecycle _ " before compiled.", !
+
+ Set tPairs = $ListBuild(
+ $LISTBUILD("%ZPM.PackageManager.Developer.Lifecycle.Module", "%IPM.Lifecycle.Module"),
+ $LISTBUILD("%ZPM.PackageManager.Developer.Processor.Abstract", "%IPM.ResourceProcessor.Abstract")
+ )
+ Set ptr = 0
+ While $ListNext(tPairs, ptr, pair) {
+ Set $ListBuild(tOldLifecycle, tNewLifecycle) = pair
+ If $$$defClassKeyGet(tClass, $$$cCLASSsuper) = tOldLifecycle {
+ $$$defClassKeySet(tClass, $$$cCLASSsuper, tNewLifecycle)
+ Write !, $$$FormattedLine($$$Magenta, "WARNING: ")
+ Write tName _ " extends the deprecated class " _ tOldLifecycle _ ". It has been updated to " _ tNewLifecycle _ " before compiled.", !
+ }
}
}
Quit $System.OBJ.CompileList(.pItems, .qSpec)
From 5c992bd8d8724c3fb1703474ab3ec881ce31414c Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Dec 2024 12:39:16 -0500
Subject: [PATCH 161/182] chore: update changelog to include `-extra-pip-flags`
---
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a34f10b7..076c3431 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #562 Added a generic resource processpor `WebApplication`, which handles creating and removal of all Security.Applications resources
- #575 Added ability to expand `$$$macro` in module.xml. The macro cannot take any arguments yet.
- #595 Added ability to bypass installation of python dependencies with -bypass-py-deps or -DBypassPyDeps=1.
+- #647 Added ability to add extra flags when installing python dependencies using pip
### Changed
- IPM is now namespace-specific rather than being installed in %SYS and being available instance-wide.
From 18599fecff33992b5e6510eadbee09b6a08abb7b Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Dec 2024 14:36:22 -0500
Subject: [PATCH 162/182] style: fix grammar mistakes
---
src/cls/IPM/Utils/LegacyCompat.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/Utils/LegacyCompat.cls b/src/cls/IPM/Utils/LegacyCompat.cls
index 98ad797b..3350a494 100644
--- a/src/cls/IPM/Utils/LegacyCompat.cls
+++ b/src/cls/IPM/Utils/LegacyCompat.cls
@@ -27,7 +27,7 @@ ClassMethod UpdateSuperclassAndCompile(ByRef pItems, ByRef qSpec = "") As %Statu
If $$$defClassKeyGet(tClass, $$$cCLASSsuper) = tOldLifecycle {
$$$defClassKeySet(tClass, $$$cCLASSsuper, tNewLifecycle)
Write !, $$$FormattedLine($$$Magenta, "WARNING: ")
- Write tName _ " extends the deprecated class " _ tOldLifecycle _ ". It has been updated to " _ tNewLifecycle _ " before compiled.", !
+ Write tName _ " extends the deprecated class " _ tOldLifecycle _ ". It has been updated to " _ tNewLifecycle _ " before compilation.", !
}
}
}
From cc64956ec9d2ffb7406c77bea44a2716f93f948b Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Thu, 5 Dec 2024 14:49:01 -0500
Subject: [PATCH 163/182] docs: list the default pip flags when install python
dependencies
---
src/cls/IPM/Main.cls | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index fd99c005..70ea42ea 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -265,7 +265,7 @@ load https://github.com/user/repository.git -branch feature-1
-
+
@@ -383,7 +383,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
-
+
@@ -414,7 +414,7 @@ run C:\Temp\MyCommands.json, where contents of the file are as follows:
-
+
From 478042dc67c47040917b3d487bed9e1f9a8cda8f Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 11 Dec 2024 10:47:31 -0500
Subject: [PATCH 164/182] fix: don't add needless mappings
This was leading to deadlock in perfectly reasonable cases and still could lead to deadlock in less-reasonable cases.
---
CHANGELOG.md | 1 +
.../IPM/ResourceProcessor/Default/Class.cls | 2 +-
.../ResourceProcessor/Default/Document.cls | 29 +++++++++----------
.../Default/LocalizedMessages.cls | 2 +-
.../IPM/ResourceProcessor/Default/Package.cls | 4 +--
.../IPM/ResourceProcessor/Default/Routine.cls | 2 +-
6 files changed, 19 insertions(+), 21 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 076c3431..541c98fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -60,6 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
- #607: Uninstall reports deletion of non-classes
- #606: Don't put garbage folders in tar archive
+- #652: Don't create extra needless mappings (could cause deadlock on multithreaded installation)
### Security
-
diff --git a/src/cls/IPM/ResourceProcessor/Default/Class.cls b/src/cls/IPM/ResourceProcessor/Default/Class.cls
index 08f7f3a4..2255d355 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Class.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Class.cls
@@ -19,7 +19,7 @@ Method OnConfigureMappings(ByRef pParams) As %Status
Set tName = $Piece(..ResourceReference.Name,".",1,*-1)
Set tGlobalScope = ..ResourceReference.Module.GlobalScope && '$Get(pParams("Reload","ForceLocalScope"),0)
If 'tGlobalScope {
- Set tNeedExplicitMapping = ..FileExistsInCurrentNS(tName)
+ Set tNeedExplicitMapping = ..ResourceIsMappedToDefaultDB(..ResourceReference.Name)
If tNeedExplicitMapping {
Set tPackage = $p(tName,".",1,*-1)
Set tSourceDB = ##class(%IPM.Utils.Module).GetRoutineDatabase($Namespace)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index 8c160553..47c19c26 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -609,23 +609,20 @@ Method OnGetUniqueName(Output pUniqueName)
/// Returns a boolean of whether current file exists in the the routine database of the current namespace exists in current namespace's database
/// Helper method to be used in derived classes' OnConfigureMappings() to skip creating unnecessary mapping
-ClassMethod FileExistsInCurrentNS(pFileName As %String) As %Boolean
+ClassMethod ResourceIsMappedToDefaultDB(pResourceName As %String) As %Boolean
{
- Set tSourceDB = ##class(%IPM.Utils.Module).GetRoutineDatabase($Namespace)
- // Check if current file exists in the the routine database of the current namespace exists in current namespace('s database)
- // If yes, skip creating unnecessary mappings
- // (Spec,Dir=1,OrderBy=1,SystemFiles=1,Flat,NotStudio=0,ShowGenerated=0,Filter,RoundTime=0,Mapped=0)
- // Set mapped=0 since we only want to check whether the file exists in the current routine DB
- Set tResult = ##class(%Library.RoutineMgr).StudioOpenDialogFunc(pFileName_".*",1,1,1,,,,,,0)
- Set tNeedExplicitMapping = 1
- While tResult.%Next(.tSC) {
- $$$ThrowOnError(tSC)
- if ($ZConvert(tResult.%Get("Name"),"U") = $ZConvert(pFileName, "U")) {
- Set tNeedExplicitMapping = 0
- }
- }
- $$$ThrowOnError(tSC)
- Return tNeedExplicitMapping
+ Set defaultDB = ##class(%IPM.Utils.Module).GetRoutineDatabaseDir($Namespace)
+ Set name = $Piece(pResourceName,".",1,*-1)
+ Set type = $ZConvert($Piece(pResourceName,".",*),"U")
+ If (type = "PKG") {
+ Set db = ##class(%SYS.Namespace).GetPackageDest(,name)
+ } ElseIf (type = "CLS") {
+ Set db = ##class(%SYS.Namespace).GetPackageDest(,$Piece(name,".",1,*-1))
+ } Else {
+ Set db = ##class(%SYS.Namespace).GetRoutineDest(,name)
+ }
+ Set db = $Piece(db,"^",2,*) // Slight difference in format of reporting here
+ Return (db = defaultDB)
}
}
diff --git a/src/cls/IPM/ResourceProcessor/Default/LocalizedMessages.cls b/src/cls/IPM/ResourceProcessor/Default/LocalizedMessages.cls
index f49ceee3..426d3ef8 100644
--- a/src/cls/IPM/ResourceProcessor/Default/LocalizedMessages.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/LocalizedMessages.cls
@@ -55,7 +55,7 @@ Method OnConfigureMappings(ByRef pParams) As %Status
Set tName = $Piece(..ResourceReference.Name,".",1,*-1)
Set tGlobalScope = ..ResourceReference.Module.GlobalScope && '$Get(pParams("Reload","ForceLocalScope"),0)
If 'tGlobalScope {
- Set tNeedExplicitMapping = ..FileExistsInCurrentNS(tName)
+ Set tNeedExplicitMapping = ..ResourceIsMappedToDefaultDB(tName_".INC")
If tNeedExplicitMapping {
Set tSourceDB = ##class(%IPM.Utils.Module).GetRoutineDatabase($Namespace)
Set tSC = ##class(%IPM.Utils.Module).AddRoutineMapping($namespace,tName,,tSourceDB)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Package.cls b/src/cls/IPM/ResourceProcessor/Default/Package.cls
index 16c876df..10247b5e 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Package.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Package.cls
@@ -16,7 +16,7 @@ Property Directory As %String(MAXLEN = "") [ InitialExpression = "cls" ];
Property LoadAsDirectory As %Boolean [ InitialExpression = 1 ];
/// Extension for individual filename(s) that comprise this resource
-Property FilenameExtension As %String [ InitialExpression = "" ];
+Property FilenameExtension As %String;
/// Subclasses may override to customize mapping behavior at the beginning of the Reload phase.
Method OnConfigureMappings(ByRef pParams) As %Status
@@ -26,7 +26,7 @@ Method OnConfigureMappings(ByRef pParams) As %Status
Set tName = $Piece(..ResourceReference.Name,".",1,*-1)
Set tGlobalScope = ..ResourceReference.Module.GlobalScope && '$Get(pParams("Reload","ForceLocalScope"),0)
If 'tGlobalScope {
- Set tNeedExplicitMapping = ..FileExistsInCurrentNS(tName)
+ Set tNeedExplicitMapping = ..ResourceIsMappedToDefaultDB(..ResourceReference.Name)
If tNeedExplicitMapping {
Set tSourceDB = ##class(%IPM.Utils.Module).GetRoutineDatabase($Namespace)
Set tSC = ##class(%IPM.Utils.Module).AddPackageMapping($namespace,tName,tSourceDB)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Routine.cls b/src/cls/IPM/ResourceProcessor/Default/Routine.cls
index 9b32e5fb..f051a7c0 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Routine.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Routine.cls
@@ -22,7 +22,7 @@ Method OnConfigureMappings(ByRef pParams) As %Status
Set tName = $Piece(..ResourceReference.Name,".",1,*-1)
Set tGlobalScope = ..ResourceReference.Module.GlobalScope && '$Get(pParams("Reload","ForceLocalScope"),0)
If 'tGlobalScope && '..LoadAsDirectory {
- Set tNeedExplicitMapping = ..FileExistsInCurrentNS(tName)
+ Set tNeedExplicitMapping = ..ResourceIsMappedToDefaultDB(..ResourceReference.Name)
If tNeedExplicitMapping {
Set tSourceDB = ##class(%IPM.Utils.Module).GetRoutineDatabase($Namespace)
Set tSC = ##class(%IPM.Utils.Module).AddRoutineMapping($namespace,tName,,tSourceDB)
From 43e99574332eb3a1e225fd1f7202a37be278c014 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 11 Dec 2024 10:50:45 -0500
Subject: [PATCH 165/182] chore (docs): tweak changelog
this is just to force CI to run...
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 541c98fa..ec7a425d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -60,7 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #559: Allow treating the "w" in SemVer x.y.z-w as a post-release rather than pre-release.
- #607: Uninstall reports deletion of non-classes
- #606: Don't put garbage folders in tar archive
-- #652: Don't create extra needless mappings (could cause deadlock on multithreaded installation)
+- #652: Don't create extra needless mappings (could cause deadlock with parallel installation of dependencies)
### Security
-
From aee3891e663bc685a3da4694d161cddc3302268a Mon Sep 17 00:00:00 2001
From: Emma Neil
Date: Thu, 31 Oct 2024 15:10:06 -0400
Subject: [PATCH 166/182] HSIEO-11006: Fix conditions for marking code as
deployed
---
CHANGELOG.md | 1 +
src/cls/IPM/Lifecycle/Base.cls | 70 ++++++++++++++++++++++++--------
src/cls/IPM/Lifecycle/Module.cls | 10 -----
src/cls/IPM/Main.cls | 18 +++++++-
src/cls/IPM/Storage/Module.cls | 12 +++++-
5 files changed, 79 insertions(+), 32 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 076c3431..190c0487 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #527: IPM 0.9.x+ ignores the casing of resources when matching files on disk even on case-sensitive filesystems
### Fixed
+- HSIEO-11006: Fix conditions for marking code as deployed
- HSIEO-9269, HSIEO-9402: % percent perforce directories are no longer necessary
- HSIEO-9269, HSIEO-9404: Repo check should happen in the order to repo creation, not by repo name
- HSIEO-9269, HSIEO-9411: Make sure can load and export xml Package-type resource
diff --git a/src/cls/IPM/Lifecycle/Base.cls b/src/cls/IPM/Lifecycle/Base.cls
index 60d71170..ab4e6d4d 100644
--- a/src/cls/IPM/Lifecycle/Base.cls
+++ b/src/cls/IPM/Lifecycle/Base.cls
@@ -55,8 +55,11 @@ Method OnAfterResourceProcessing(pPhase As %String, ByRef pParams) As %Status
Method %DispatchMethod(pMethod As %String, ByRef pParams, Args...) [ ServerOnly = 1 ]
{
- if $listfind(..#PHASES,pMethod)=0 do $zu(96,3,$$$ERNOMETHOD,1,"","method "_pMethod_" of class "_$classname())
- quit ..Module.ExecutePhases(..Module.Name,$lb(pMethod),1,.pParams)
+ // Match method name with CamelCased lifecycle method
+ set convertedMethod = ..MatchSinglePhase(pMethod)
+
+ if $listfind(..#PHASES,convertedMethod)=0 do $zu(96,3,$$$ERNOMETHOD,1,"","method "_convertedMethod_" of class "_$classname())
+ quit ..Module.ExecutePhases(..Module.Name,$lb(convertedMethod),1,.pParams)
}
/// Merges default parameters into pParams
@@ -131,22 +134,48 @@ ClassMethod GetCompletePhases(pPhases As %List) As %List
/// This method defines what a complete phase means for a given phase
ClassMethod GetCompletePhasesForOne(pOnePhase As %String) As %List
{
+ set pOnePhase = $ZCONVERT(pOnePhase, "L")
+
+ Quit $Case(pOnePhase,
+ "clean": $ListBuild("Clean"),
+ "reload": $ListBuild("Reload","*"),
+ "validate": $ListBuild("Reload","*","Validate"),
+ "exportdata": $ListBuild("ExportData"),
+ "compile": $ListBuild("Reload","*","Validate","Compile"),
+ "activate": $ListBuild("Reload","*","Validate","Compile","Activate"),
+ "document": $ListBuild("Document"),
+ "makedeployed": $ListBuild("MakeDeployed"),
+ "test": $ListBuild("Reload","*","Validate","Compile","Activate","Test"),
+ "package": $ListBuild("Reload","*","Validate","Compile","Activate","Package"),
+ "verify": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Verify"),
+ "register": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register"),
+ "publish": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
+ "configure": $ListBuild("Configure"),
+ "unconfigure": $ListBuild("Unconfigure"),
+ : ""
+ )
+}
+
+/// Match single inputted phase to the correctly CamelCased lifecycle phase
+ClassMethod MatchSinglePhase(pOnePhase As %String) As %String
+{
+ set pOnePhase = $ZCONVERT(pOnePhase, "L")
Quit $Case(pOnePhase,
- "Clean": $ListBuild("Clean"),
- "Reload": $ListBuild("Reload","*"),
- "Validate": $ListBuild("Reload","*","Validate"),
- "ExportData": $ListBuild("ExportData"),
- "Compile": $ListBuild("Reload","*","Validate","Compile"),
- "Activate": $ListBuild("Reload","*","Validate","Compile","Activate"),
- "Document": $ListBuild("Document"),
- "MakeDeployed": $ListBuild("MakeDeployed"),
- "Test": $ListBuild("Reload","*","Validate","Compile","Activate","Test"),
- "Package": $ListBuild("Reload","*","Validate","Compile","Activate","Package"),
- "Verify": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Verify"),
- "Register": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register"),
- "Publish": $ListBuild("Reload","*","Validate","Compile","Activate","Package","Register","Publish"),
- "Configure": $ListBuild("Configure"),
- "Unconfigure": $ListBuild("Unconfigure"),
+ "clean": "Clean",
+ "reload": "Reload",
+ "validate": "Validate",
+ "exportdata": "ExportData",
+ "compile": "Compile",
+ "activate": "Activate",
+ "document": "Document",
+ "makedeployed": "MakeDeployed",
+ "test": "Test",
+ "package": "Package",
+ "verify": "Verify",
+ "register": "Register",
+ "publish": "Publish",
+ "configure": "Configure",
+ "unconfigure": "Unconfigure",
: ""
)
}
@@ -1542,13 +1571,18 @@ Method %MakeDeployed(ByRef pParams) As %Status
Try {
Set tDev = ..Module.DeveloperMode
Set tVerbose = $Get(pParams("Verbose"))
+
+ // If the recurse parameter is set, then this lifecycle method should recursively deploy all of the
+ // module's dependencies marked for deployment
+ Set tLockedDependencies = $Get(pParams("Recurse"),0) // Indicates whether entire dependency graph should be traversed and deployed
+
If tDev && tVerbose {
Write !,"Module is in developer mode; will only report what WOULD be deployed unless packaging, in which case items WILL be deployed."
}
// Default implementation: see which resources are expicitly flagged with Deploy = true.
// Build an array of those, then mark them as deployed.
- $$$ThrowOnError(..Module.GetResolvedReferences(.tResourceArray,1,..PhaseList,1,.pDependencyGraph))
+ $$$ThrowOnError(..Module.GetResolvedReferences(.tResourceArray,tLockedDependencies,..PhaseList,1,.pDependencyGraph))
Set tResourceKey = ""
For {
diff --git a/src/cls/IPM/Lifecycle/Module.cls b/src/cls/IPM/Lifecycle/Module.cls
index 128130c1..b5790a89 100644
--- a/src/cls/IPM/Lifecycle/Module.cls
+++ b/src/cls/IPM/Lifecycle/Module.cls
@@ -38,16 +38,6 @@ Method %Activate(ByRef pParams) As %Status
Set tSC = ..Configure(.pParams)
$$$ThrowOnError(tSC)
-
- If '$ListFind(..PhaseList,"Package") {
- // Code cannot be deployed if it is to be reexported and packaged.
- Set tDevMode = $Get(pParams("DeveloperMode"), ..Module.DeveloperMode)
- Set isKitBuild = $Get(pParams("IsKitBuild"),0)
- If ('tDevMode && isKitBuild) {
- Set tSC = ..MakeDeployed(.pParams)
- $$$ThrowOnError(tSC)
- }
- }
// Create Studio project for package if it is loaded in developer mode and no explicit statement to not create it
Set tNoStudioProject = $Get(pParams("NoStudioProject"), 0)
diff --git a/src/cls/IPM/Main.cls b/src/cls/IPM/Main.cls
index 70ea42ea..df96edb8 100644
--- a/src/cls/IPM/Main.cls
+++ b/src/cls/IPM/Main.cls
@@ -7,7 +7,7 @@ Class %IPM.Main Extends %IPM.CLI
Parameter DOMAIN = "ZPM";
-Parameter STANDARDPHASES = {$ListBuild("reload","compile","test","package","verify","publish")};
+Parameter STANDARDPHASES = {$ListBuild("reload","compile","test","package","verify","publish","makedeployed")};
/// Description of commands to use for this CLI
XData Commands [ XMLNamespace = "http://www.intersystems.com/PackageManager/CLI" ]
@@ -45,6 +45,7 @@ resources exported to the filesystem (and possible to source control) are consis
with what is in the database.
* compile: compiles all resources within the module.
* activate: performs post-compilation installation/configuration steps.
+* makedeployed: deploys resources within the module for which deployment is enabled.
* document: regenerates the API documentation for the module
* test: runs any unit tests associated with the module, in the current namespace.
* package: exports the module's resources and bundles them into a module artifact (.tgz file).
@@ -156,6 +157,18 @@ This command is an alias for `module-action module-name publish`
+
+
+This command is an alias for `module-action module-name makedeployed`
+
+
+
+
+
+
+
+
+
Delete package from registry
@@ -2132,7 +2145,8 @@ ClassMethod RunOnePhase(ByRef pCommandInfo) [ Internal ]
Set tModName = $Get(pCommandInfo("parameters","module"))
Set tPhases = $ListBuild($ZConvert(pCommandInfo, "w"))
Set tIsComplete = '$$$HasModifier(pCommandInfo,"only")
- Set tParams("cmd") = pCommandInfo
+ Set tParams("Recurse") = $$$HasModifier(pCommandInfo,"recurse")
+ Set tParams("cmd") = pCommandInfo
Merge tParams = pCommandInfo("data")
$$$ThrowOnError(##class(%IPM.Storage.Module).ExecutePhases(tModName,tPhases,tIsComplete,.tParams))
}
diff --git a/src/cls/IPM/Storage/Module.cls b/src/cls/IPM/Storage/Module.cls
index d2b7f71e..b121dcae 100644
--- a/src/cls/IPM/Storage/Module.cls
+++ b/src/cls/IPM/Storage/Module.cls
@@ -238,7 +238,7 @@ ClassMethod ExecutePhases(pModuleName As %String, pPhases As %List, pIsComplete
Set tPhases = $ListBuild("PrepareDeploy") _ tPhases
}
} Else {
- Set tPhases = pPhases
+ Set tPhases = $ListBuild(##class(%IPM.Lifecycle.Base).MatchSinglePhase($LISTTOSTRING(pPhases)))
}
// Lifecycle-provided default parameters
@@ -309,7 +309,7 @@ ClassMethod ExecutePhases(pModuleName As %String, pPhases As %List, pIsComplete
Quit
}
If $IsObject(tResource.Processor) {
- Do tResource.Processor.SetParams(.pParams)
+ Do tResource.Processor.SetParams(.pParams)
Set tSC = $Method(tResource.Processor,"OnBeforePhase",tOnePhase,.pParams)
$$$ThrowOnError(tSC)
}
@@ -480,6 +480,7 @@ Method GetDefaultParameters(Output pParams)
}
}
+/// Returns whether pScope is in the list of pPhases
ClassMethod HasScope(pPhases As %List, pScope As %String) [ Private ]
{
If (pScope = "") {
@@ -917,6 +918,13 @@ ClassMethod GetKnownDependencies(pModuleName As %String) As %List
Quit tKnownDependencyList
}
+/// Builds a module's immediate dependency graph and array of resources.
+/// Optionally loads uninstalled dependency modules and recurses over each module in the dependency graph.
+/// @Argument pReferenceArray Array of all module's resources (including resources that compose a resource) that contain the appropriate phase scope.
+/// @Argument pLockedDependencies Whether method should be recursively applied to the module's dependencies (true = yes).
+/// @Argument pPhases List of IPM lifecycle phases to be applied to the current module.
+/// @Argument pSkipDependencies Whether to skip loading uninstalled dependency modules.
+/// @Argument pDependencyGraph Tree of module's dependencies.
Method GetResolvedReferences(Output pReferenceArray, pLockedDependencies As %Boolean = 0, pPhases As %List = "", pSkipDependencies As %Boolean = 0, ByRef pDependencyGraph) As %Status
{
Set tSC = $$$OK
From 4f771561a2ae94ec15a65b7cebad4b779f0571e7 Mon Sep 17 00:00:00 2001
From: isc-tleavitt <73311181+isc-tleavitt@users.noreply.github.com>
Date: Wed, 11 Dec 2024 11:04:06 -0500
Subject: [PATCH 167/182] chore (docs): update method description
---
src/cls/IPM/ResourceProcessor/Default/Document.cls | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/Default/Document.cls b/src/cls/IPM/ResourceProcessor/Default/Document.cls
index 47c19c26..533b0daa 100644
--- a/src/cls/IPM/ResourceProcessor/Default/Document.cls
+++ b/src/cls/IPM/ResourceProcessor/Default/Document.cls
@@ -607,8 +607,8 @@ Method OnGetUniqueName(Output pUniqueName)
}
}
-/// Returns a boolean of whether current file exists in the the routine database of the current namespace exists in current namespace's database
-/// Helper method to be used in derived classes' OnConfigureMappings() to skip creating unnecessary mapping
+/// Helper method to be used in derived classes' OnConfigureMappings() to skip creating unnecessary mappings.
+/// Returns true if pResourceName (in InternalName format - e.g., %Foo.Bar.PKG) is mapped to the current namespace's default routine database.
ClassMethod ResourceIsMappedToDefaultDB(pResourceName As %String) As %Boolean
{
Set defaultDB = ##class(%IPM.Utils.Module).GetRoutineDatabaseDir($Namespace)
From e8d22bd692b6f6b1b47c751af4423c4977b857e9 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Fri, 13 Dec 2024 09:28:14 -0500
Subject: [PATCH 168/182] Updating FileCopy to check for $ variables in path
---
src/cls/IPM/ResourceProcessor/FileCopy.cls | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls
index 7be88a24..3124301c 100644
--- a/src/cls/IPM/ResourceProcessor/FileCopy.cls
+++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls
@@ -39,7 +39,12 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
}
If (pPhase = "Activate") && (..InstallDirectory '= "") && ('..Defer) {
- Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
+ if ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") {
+ Set tSource = $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
+ }
+ else {
+ Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
+ }
Set tTarget = ..InstallDirectory
Set tSC = ..DoCopy(tSource, tTarget, .pParams)
If $$$ISERR(tSC) {
@@ -65,7 +70,12 @@ Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
}
If (pPhase = "Activate") && (..InstallDirectory '= "") && (..Defer) {
- Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
+ if ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") {
+ Set tSource = $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
+ }
+ else {
+ Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
+ }
Set tTarget = ..InstallDirectory
Set tSC = ..DoCopy(tSource, tTarget, .pParams)
If $$$ISERR(tSC) {
From abb9659a83c249fe6e279d5db7fa1f155282158c Mon Sep 17 00:00:00 2001
From: "Shuheng Liu (InterSystems)"
<155992272+isc-shuliu@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:13:11 -0500
Subject: [PATCH 169/182] chore: update branch names in main.yml (v0.9.x)
---
.github/workflows/main.yml | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0c1df545..0b0fdab6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,12 +2,14 @@ name: CI
on:
push:
branches:
- - master
- - v1
+ - v0.7.x
+ - v0.9.x
+ - v0.10.x
pull_request:
branches:
- - master
- - v1
+ - v0.7.x
+ - v0.9.x
+ - v0.10.x
release:
types:
- released
@@ -302,4 +304,4 @@ jobs:
git add module.xml
git commit -m 'auto bump version with release'
git push
- )
\ No newline at end of file
+ )
From 4cc8f14e5daeba8fe6e8aef8884b20ed1d9b2b49 Mon Sep 17 00:00:00 2001
From: "Shuheng Liu (InterSystems)"
<155992272+isc-shuliu@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:38:35 -0500
Subject: [PATCH 170/182] fix(ci): bump legacy zpm version
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0b0fdab6..e9f9a32e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -172,7 +172,7 @@ jobs:
run: |
curl http://localhost:52773/registry/packages/-/all | jq
curl http://localhost:52773/registry/packages/zpm/ | jq
- ASSET_NAME='zpm-0.7.3.xml'
+ ASSET_NAME='zpm-0.7.4.xml'
ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
wget $ASSET_URL -O /tmp/zpm.xml
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
From 9ac597eca3c1ec3aeac5af7f74fcf3b52fd7e328 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Fri, 13 Dec 2024 14:31:22 -0500
Subject: [PATCH 171/182] Minor update to FileCopy if statement + adding to
changelog
---
.github/workflows/main.yml | 2 +-
CHANGELOG.md | 1 +
src/cls/IPM/ResourceProcessor/FileCopy.cls | 61 +++++++++++++++-------
3 files changed, 44 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0c1df545..51a8de0f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -170,7 +170,7 @@ jobs:
run: |
curl http://localhost:52773/registry/packages/-/all | jq
curl http://localhost:52773/registry/packages/zpm/ | jq
- ASSET_NAME='zpm-0.7.3.xml'
+ ASSET_NAME='zpm-0.7.4.xml'
ASSET_URL=`wget --header "Authorization: token ${GITHUB_TOKEN}" -qO- https://api.github.com/repos/intersystems/ipm/releases | jq -r ".[].assets[] | select(.name == \"${ASSET_NAME}\") | .browser_download_url"`
wget $ASSET_URL -O /tmp/zpm.xml
CONTAINER=$(docker run --network zpm --rm -d ${{ steps.image.outputs.name }} ${{ steps.image.outputs.flags }})
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cd6b4020..ab5ca72b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #527: IPM 0.9.x+ ignores the casing of resources when matching files on disk even on case-sensitive filesystems
### Fixed
+- HSIEO-10884: Bug Fix - FileCopy to check for $ variables in path
- HSIEO-11006: Fix conditions for marking code as deployed
- HSIEO-9269, HSIEO-9402: % percent perforce directories are no longer necessary
- HSIEO-9269, HSIEO-9404: Repo check should happen in the order to repo creation, not by repo name
diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls
index 3124301c..2b17dc75 100644
--- a/src/cls/IPM/ResourceProcessor/FileCopy.cls
+++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls
@@ -27,7 +27,9 @@ Property CSPApplication As %String(MAXLEN = "");
/// Use this for build artifacts.
Property Defer As %Boolean [ InitialExpression = 0 ];
-Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
+Method OnBeforePhase(
+ pPhase As %String,
+ ByRef pParams) As %Status
{
Set tVerbose = $Get(pParams("Verbose"))
// Default implementation: call %ValidateObject to validate attributes
@@ -39,12 +41,7 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
}
If (pPhase = "Activate") && (..InstallDirectory '= "") && ('..Defer) {
- if ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") {
- Set tSource = $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
- }
- else {
- Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
- }
+ Set tSource = ..GetSource()
Set tTarget = ..InstallDirectory
Set tSC = ..DoCopy(tSource, tTarget, .pParams)
If $$$ISERR(tSC) {
@@ -57,7 +54,9 @@ Method OnBeforePhase(pPhase As %String, ByRef pParams) As %Status
Quit tSC
}
-Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
+Method OnAfterPhase(
+ pPhase As %String,
+ ByRef pParams) As %Status
{
Set tVerbose = $Get(pParams("Verbose"))
// Default implementation: call %ValidateObject to validate attributes
@@ -70,12 +69,7 @@ Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
}
If (pPhase = "Activate") && (..InstallDirectory '= "") && (..Defer) {
- if ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") {
- Set tSource = $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
- }
- else {
- Set tSource = ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
- }
+ Set tSource = ..GetSource()
Set tTarget = ..InstallDirectory
Set tSC = ..DoCopy(tSource, tTarget, .pParams)
If $$$ISERR(tSC) {
@@ -88,7 +82,10 @@ Method OnAfterPhase(pPhase As %String, ByRef pParams) As %Status
Quit tSC
}
-Method OnBeforeArtifact(pExportDirectory As %String, pWorkingDirectory As %String, ByRef pParams) As %Status
+Method OnBeforeArtifact(
+ pExportDirectory As %String,
+ pWorkingDirectory As %String,
+ ByRef pParams) As %Status
{
Set tSC = $$$OK
Try {
@@ -108,7 +105,11 @@ Method OnBeforeArtifact(pExportDirectory As %String, pWorkingDirectory As %Strin
Quit tSC
}
-Method NormalizeNames(ByRef pSource As %String, ByRef pTarget As %String, Output pTargetDir, Output pAsFile As %Boolean)
+Method NormalizeNames(
+ ByRef pSource As %String,
+ ByRef pTarget As %String,
+ Output pTargetDir,
+ Output pAsFile As %Boolean)
{
Set pAsFile = 0
If ("\/"[$Extract(pSource, *))
@@ -131,7 +132,21 @@ Method NormalizeNames(ByRef pSource As %String, ByRef pTarget As %String, Output
}
}
-Method DoCopy(tSource, tTarget, pParams)
+Method GetSource()
+{
+ // We don't want to prefix if the path starts with a $ variable
+ if (($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") || ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "${")) {
+ return $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
+ }
+ else {
+ return ..ResourceReference.Module.Root _ $Case(..SourceDirectory, "": ..ResourceReference.Name,: ..SourceDirectory)
+ }
+}
+
+Method DoCopy(
+ tSource,
+ tTarget,
+ pParams)
{
Set tVerbose = $Get(pParams("Verbose"))
Set tSC = $$$OK
@@ -167,7 +182,12 @@ Method DoCopy(tSource, tTarget, pParams)
Quit tSC
}
-Method OnExportItem(pFullExportPath As %String, pItemName As %String, ByRef pItemParams, ByRef pParams, Output pItemHandled As %Boolean = 0) As %Status
+Method OnExportItem(
+ pFullExportPath As %String,
+ pItemName As %String,
+ ByRef pItemParams,
+ ByRef pParams,
+ Output pItemHandled As %Boolean = 0) As %Status
{
Set tVerbose = $Get(pParams("Verbose"))
@@ -180,7 +200,10 @@ Method OnExportItem(pFullExportPath As %String, pItemName As %String, ByRef pIte
Quit ..DoCopy(tSource, tTarget, .pParams)
}
-Method OnPhase(pPhase As %String, ByRef pParams, Output pResourceHandled As %Boolean = 0) As %Status
+Method OnPhase(
+ pPhase As %String,
+ ByRef pParams,
+ Output pResourceHandled As %Boolean = 0) As %Status
{
Set tVerbose = $Get(pParams("Verbose"))
If (pPhase = "Clean") {
From 2b4f85f7f1f159610496f0635121e53cb245c7e6 Mon Sep 17 00:00:00 2001
From: James Lechtner
Date: Fri, 13 Dec 2024 16:03:11 -0500
Subject: [PATCH 172/182] Updating method description comment
---
src/cls/IPM/ResourceProcessor/FileCopy.cls | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/cls/IPM/ResourceProcessor/FileCopy.cls b/src/cls/IPM/ResourceProcessor/FileCopy.cls
index 2b17dc75..579244ef 100644
--- a/src/cls/IPM/ResourceProcessor/FileCopy.cls
+++ b/src/cls/IPM/ResourceProcessor/FileCopy.cls
@@ -132,9 +132,9 @@ Method NormalizeNames(
}
}
+/// Prefixes the path to the module root if it does not start with a $ variable
Method GetSource()
{
- // We don't want to prefix if the path starts with a $ variable
if (($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "{$") || ($Extract(..ResourceReference.Attributes.GetAt("SourceDirectory"), 0, 2) = "${")) {
return $Case(..SourceDirectory, "": ..ResourceReference.Module.Root _ ..ResourceReference.Name,: ..SourceDirectory)
}
From abd9f796caa42028decf00470ee867f7ea5718f6 Mon Sep 17 00:00:00 2001
From: "Shuheng Liu (InterSystems)"
<155992272+isc-shuliu@users.noreply.github.com>
Date: Mon, 16 Dec 2024 09:23:49 -0500
Subject: [PATCH 173/182] chore(ci): include `main` branch for CI
---
.github/workflows/main.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index e9f9a32e..a28d2004 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,11 +2,13 @@ name: CI
on:
push:
branches:
+ - main
- v0.7.x
- v0.9.x
- v0.10.x
pull_request:
branches:
+ - main
- v0.7.x
- v0.9.x
- v0.10.x
From 5025c10eea36006a69f935b23195c979ef7bc4f7 Mon Sep 17 00:00:00 2001
From: "Shuheng Liu (InterSystems)"
<155992272+isc-shuliu@users.noreply.github.com>
Date: Mon, 16 Dec 2024 09:30:40 -0500
Subject: [PATCH 174/182] fix(ci): change ref from master to main on release
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a28d2004..a66b999d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -241,7 +241,7 @@ jobs:
- uses: actions/checkout@master
if: github.event_name == 'release'
with:
- ref: master
+ ref: main
- uses: actions/download-artifact@v3
with:
name: zpm-${{ needs.prepare.outputs.version }}
From 3519d32f391043ed30040df3b3d6bf7c6a0c9004 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 16 Dec 2024 10:11:09 -0500
Subject: [PATCH 175/182] chore: update module.xml and changelog for 0.9.0
release
---
CHANGELOG.md | 8 +-------
module.xml | 2 +-
2 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ab5ca72b..9c8d5dea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased - 0.9.0+snapshot]
+## [0.9.0] - 2024-12-16
### Added
- #364 Added ability to restrict the installation to IRIS or IRIS for Health platform to the SystemRequirements attribute
@@ -64,11 +64,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- #606: Don't put garbage folders in tar archive
- #652: Don't create extra needless mappings (could cause deadlock with parallel installation of dependencies)
-### Security
--
-
-### Removed
--
-
### Deprecated
- #593 CSPApplication is deprecated in favor of WebApplication. User will be warned when installing a package containing CSPApplication.
diff --git a/module.xml b/module.xml
index 45e95440..e0bcdb6c 100644
--- a/module.xml
+++ b/module.xml
@@ -2,7 +2,7 @@
ZPM
- 0.9.0-SNAPSHOT
+ 0.9.0
Package Management System
Provides development tools and infrastructure for defining, building, distributing, and installing modules and applications.
Package Manager
From 2e6cd52ceb80f01a91c551a3a3cf22848b77541d Mon Sep 17 00:00:00 2001
From: ProjectBot
Date: Mon, 16 Dec 2024 19:36:10 +0000
Subject: [PATCH 176/182] auto bump version with release
---
module.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/module.xml b/module.xml
index e0bcdb6c..4d8205d7 100644
--- a/module.xml
+++ b/module.xml
@@ -2,7 +2,7 @@
ZPM
- 0.9.0
+ 0.9.1-SNAPSHOT
Package Management System
Provides development tools and infrastructure for defining, building, distributing, and installing modules and applications.
Package Manager
From 5b707c0d973d52ded7bc7b5fdedd9fb1465e31a3 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Mon, 16 Dec 2024 15:01:58 -0500
Subject: [PATCH 177/182] fix(ci): update path to preload Installer.cls
---
.github/workflows/main.yml | 2 +-
module.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a66b999d..28e2dcbd 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -275,7 +275,7 @@ jobs:
CONTAINER=$(docker run -d --rm -v $(pwd):/home/irisowner/zpm/ containers.intersystems.com/intersystems/${{ needs.prepare.outputs.main }} --check-caps false)
sleep 5; docker exec $CONTAINER /usr/irissys/dev/Cloud/ICM/waitISC.sh
docker exec -i $CONTAINER iris session iris -UUSER << EOF
- set sc=##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm/Installer.cls","ck")
+ set sc=##class(%SYSTEM.OBJ).Load("/home/irisowner/zpm/preload/cls/IPM/Installer.cls","ck")
set sc=##class(IPM.Installer).setup("/home/irisowner/zpm/",3)
zpm "repo -r -name registry -url ""https://pm.community.intersystems.com/"" -username ${{ secrets.REGISTRY_USERNAME }} -password ${{ secrets.REGISTRY_PASSWORD }}":1
zpm "publish zpm -v":1
diff --git a/module.xml b/module.xml
index 4d8205d7..e0bcdb6c 100644
--- a/module.xml
+++ b/module.xml
@@ -2,7 +2,7 @@
ZPM
- 0.9.1-SNAPSHOT
+ 0.9.0
Package Management System
Provides development tools and infrastructure for defining, building, distributing, and installing modules and applications.
Package Manager
From 1c760de5698b50eff7568efb71263b91708cff8f Mon Sep 17 00:00:00 2001
From: ProjectBot
Date: Mon, 16 Dec 2024 20:42:24 +0000
Subject: [PATCH 178/182] auto bump version with release
---
module.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/module.xml b/module.xml
index e0bcdb6c..4d8205d7 100644
--- a/module.xml
+++ b/module.xml
@@ -2,7 +2,7 @@
ZPM
- 0.9.0
+ 0.9.1-SNAPSHOT
Package Management System
Provides development tools and infrastructure for defining, building, distributing, and installing modules and applications.
Package Manager
From 2ae9b3aecb382d27adc0d90fb16e6e04f51c94b7 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Dec 2024 16:03:38 -0500
Subject: [PATCH 179/182] debug: ci is passing locally but fails on github
---
.github/workflows/main.yml | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6e10e47f..58d519cf 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -118,8 +118,8 @@ jobs:
echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl`
docker exec -i $CONTAINER iris session iris -UUSER << EOF
zpm "list":1
- zpm "test zpm -v -only":1
- zpm "verify zpm -v -only":1
+ # zpm "test zpm -v -only":1
+ # zpm "verify zpm -v -only":1
halt
EOF
docker stop $CONTAINER
@@ -202,8 +202,12 @@ jobs:
zpm "repo -list"
zpm "search":1
zpm "install sslclient":1
+ write "Before migration",!
+ zpm "list"
zpm "install zpm -v":1
zpm "load -dev /tmp/test-package/":1
+ write "After migration",!
+ zpm "list"
zpm "test ipm-migration-v0.7-to-v0.9 -only -verbose":1
halt
EOF
From 17cbf8572a8ee8f6ce665adba2c911abdcb89e9b Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Dec 2024 16:11:39 -0500
Subject: [PATCH 180/182] fix: run migration only once
---
src/cls/IPM/Utils/Migration.cls | 28 +++++-----------------------
1 file changed, 5 insertions(+), 23 deletions(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index 65573067..d36384fc 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -5,30 +5,12 @@ Class %IPM.Utils.Migration
ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
- Set tOriginalNS = $Namespace
- Do ##class(%IPM.Main).GetListNamespace(.list)
- New $Namespace
- Set $Namespace = "%SYS"
- Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(tOriginalNS, "%IPM")
-
- Set ns = ""
Set sc = $$$OK
- For {
- Set ns = $Order(list(ns))
- // Perform migration for namespaces to which %IPM is mapped from the current namespace's default routine database
- If ##class(%SYS.Namespace).GetPackageDest(ns, "%IPM") '= sourceDB {
- Continue
- }
- If ns = "" {
- Quit
- }
- Try {
- Set $Namespace = $Zstrip(ns,"<>WC")
- Do ..MigrateZPMToIPM(verbose)
- Do ..MigrateReposFromIPM09(verbose)
- } Catch e {
- Set sc = $$$ADDSC(sc, e.AsStatus())
- }
+ Try {
+ Do ..MigrateZPMToIPM(verbose)
+ Do ..MigrateReposFromIPM09(verbose)
+ } Catch e {
+ Set sc = e.AsStatus()
}
Quit sc
}
From 94229716b93e821d8b784c01f5525f2d4f8c5bdc Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Tue, 17 Dec 2024 16:17:42 -0500
Subject: [PATCH 181/182] Revert "fix: run migration only once"
This reverts commit 17cbf8572a8ee8f6ce665adba2c911abdcb89e9b.
---
src/cls/IPM/Utils/Migration.cls | 28 +++++++++++++++++++++++-----
1 file changed, 23 insertions(+), 5 deletions(-)
diff --git a/src/cls/IPM/Utils/Migration.cls b/src/cls/IPM/Utils/Migration.cls
index d36384fc..65573067 100644
--- a/src/cls/IPM/Utils/Migration.cls
+++ b/src/cls/IPM/Utils/Migration.cls
@@ -5,12 +5,30 @@ Class %IPM.Utils.Migration
ClassMethod RunAll(verbose As %Boolean = 1) As %Status
{
+ Set tOriginalNS = $Namespace
+ Do ##class(%IPM.Main).GetListNamespace(.list)
+ New $Namespace
+ Set $Namespace = "%SYS"
+ Set sourceDB = ##class(%SYS.Namespace).GetPackageDest(tOriginalNS, "%IPM")
+
+ Set ns = ""
Set sc = $$$OK
- Try {
- Do ..MigrateZPMToIPM(verbose)
- Do ..MigrateReposFromIPM09(verbose)
- } Catch e {
- Set sc = e.AsStatus()
+ For {
+ Set ns = $Order(list(ns))
+ // Perform migration for namespaces to which %IPM is mapped from the current namespace's default routine database
+ If ##class(%SYS.Namespace).GetPackageDest(ns, "%IPM") '= sourceDB {
+ Continue
+ }
+ If ns = "" {
+ Quit
+ }
+ Try {
+ Set $Namespace = $Zstrip(ns,"<>WC")
+ Do ..MigrateZPMToIPM(verbose)
+ Do ..MigrateReposFromIPM09(verbose)
+ } Catch e {
+ Set sc = $$$ADDSC(sc, e.AsStatus())
+ }
}
Quit sc
}
From 33859a617dfd1de16525687285e74efe7a26e373 Mon Sep 17 00:00:00 2001
From: Shuheng Liu
Date: Wed, 18 Dec 2024 11:34:45 -0500
Subject: [PATCH 182/182] test: move ZPM migration test to %SYS
---
.github/workflows/main.yml | 4 ++--
.../tests/unit_tests/Test/PM/Migration/ModuleStorage.cls | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 58d519cf..b56c90a9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -118,8 +118,8 @@ jobs:
echo `docker exec -i --workdir /home/irisowner/zpm/ $CONTAINER ls -rtl`
docker exec -i $CONTAINER iris session iris -UUSER << EOF
zpm "list":1
- # zpm "test zpm -v -only":1
- # zpm "verify zpm -v -only":1
+ zpm "test zpm -v -only":1
+ zpm "verify zpm -v -only":1
halt
EOF
docker stop $CONTAINER
diff --git a/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/ModuleStorage.cls b/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/ModuleStorage.cls
index 69d4d0dc..52a2dc85 100644
--- a/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/ModuleStorage.cls
+++ b/tests/migration/v0.7-to-v0.9/tests/unit_tests/Test/PM/Migration/ModuleStorage.cls
@@ -27,6 +27,8 @@ Method TestDSW()
Method TestZPM()
{
+ New $Namespace
+ Set $Namespace = "%SYS"
Do $$$AssertTrue(##class(%IPM.Storage.Module).NameExists("zpm"))
Do $$$AssertTrue(##class(%IPM.StudioDocument.Module).Exists("zpm.ZPM"))
}