feat(replication): enhance course and category management

Implement enhanced category and course replication, improving the handling of both by ensuring new and existing categories are processed correctly. Introduce CLI support for bulk category imports using a dedicated flag and provide helper functions like `generateRandomString` and `removePath` for file management. Add a new `trigger_all.php` script to trigger replication for all courses and an `unenroll.php` script for unenrolling users with cleanup of related data.

Includes error handling improvements and directory structure checks.

import.php

@@ -3,9 +3,78 @@ require_once("../../config.php");
 require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 require_once($CFG->dirroot . "/backup/util/includes/restore_includes.php");
 
+$timestamp = $_GET["timestamp"];
+
+$replicationconfig = get_config('local_replication');
+$directory = $replicationconfig->directory;
+
+function local_replication_process_category($category, $parent_id = 0) {
+    global $DB;
+
+    $oldid = (int)$category['oldid'];
+    $name = (string)$category['name'];
+
+    $existing_category = \core_course_category::get($oldid, IGNORE_MISSING);
+
+    if ($existing_category->id) {
+        // Update name and parent if necessary
+        $updated_data = [];
+        if ($existing_category->name !== $name) {
+            $updated_data['name'] = $name;
+        }
+        if ($parent_id !== 0 && $existing_category->parent != $parent_id) {
+            $updated_data['parent'] = $parent_id;
+        }
+
+        if (!empty($updated_data)) {
+            // Update category
+            $existing_category->update($updated_data);
+        }
+    } else {
+        // Create categories up to the required ID
+        $max_id = $DB->get_field_sql("SELECT MAX(id) FROM {course_categories}");
+        while ($max_id < $oldid) {
+            $new_category_data = [
+                'name' => "Placeholder " . ($max_id + 1),
+                'parent' => $parent_id,
+                'idnumber' => '',
+                'visible' => 1
+            ];
+
+            // Create placeholder category
+            \core_course_category::create($new_category_data);
+            $max_id = $DB->get_field_sql("SELECT MAX(id) FROM {course_categories}");
+        }
+
+        // Create target category
+        $new_category_data = [
+            'id' => $oldid,
+            'name' => $name,
+            'parent' => $parent_id,
+            'idnumber' => '',
+            'visible' => 1
+        ];
+        \core_course_category::update($new_category_data);
+    }
+
+    // Process sub-categories
+    foreach ($category->category as $sub_category) {
+        local_replication_process_category($sub_category, $oldid);
+    }
+}
+
+if (isset($_GET["categories"])) {
+    $infile = $directory . DIRECTORY_SEPARATOR . "categories_" . $timestamp . ".xml";
+    $xml = simplexml_load_file($infile);
+    foreach ($xml->category as $category) {
+        local_replication_process_category($category);
+    }    
+    echo("OK");
+    exit();
+}
+
 $courseid = $_GET["courseid"];
 $categoryid = $_GET["categoryid"];
-$timestamp = $_GET["timestamp"];
 
 try {
     $course = get_course($courseid);
@@ -25,9 +94,6 @@ if ($course && !$categoryid) {
 
 $category = $DB->get_record('course_categories', array('id' => $categoryid), '*', MUST_EXIST);
 
-$replicationconfig = get_config('local_replication');
-$directory = $replicationconfig->directory;
-
 $infile = $directory . DIRECTORY_SEPARATOR . "course_" . $courseid . "_" . $categoryid . "_" . $timestamp . ".mbz";
 
 if (empty($CFG->tempdir)) {

import_cli.php

@@ -5,49 +5,6 @@ require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 require_once($CFG->dirroot . "/backup/util/includes/restore_includes.php");
 require_once($CFG->libdir . '/clilib.php');
 
-$usage = "Import a course from ContentMonster
-
-Usage:
-    # php import_cli.php --courseid=<value> --categoryid=<value> --timestamp=<value> [--source=<value>]
-    # php import_cli.php [--help|-h]
-
-Options:
-    -h --help               Print this help.
-    --paramname=<value>     Describe the parameter and the meaning of its values.
-";
-
-list($options, $unrecognised) = cli_get_params([
-    'help' => false,
-    'courseid' => null,
-    'categoryid' => null,
-    'timestamp' => null,
-    'source' => null,
-], [
-    'h' => 'help'
-]);
-
-if ($unrecognised) {
-    $unrecognised = implode(PHP_EOL . '  ', $unrecognised);
-    cli_error(get_string('cliunknowoption', 'core_admin', $unrecognised));
-}
-
-if ($options['help']) {
-    cli_writeln($usage);
-    exit(2);
-}
-
-if (empty($options['courseid'])) {
-    cli_error('Missing mandatory argument courseid.', 2);
-}
-
-if (empty($options['categoryid'])) {
-    cli_error('Missing mandatory argument categoryid.', 2);
-}
-
-if (empty($options['timestamp'])) {
-    cli_error('Missing mandatory argument timestamp.', 2);
-}
-
 function generateRandomString($length = 10) {
     $characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
     $charactersLength = strlen($characters);
@@ -76,21 +33,137 @@ function removePath($path) {
         closedir($handler);
 
         if (!rmdir($path)) {
-            trigger_error('File Error: Failed to remove folder ' . $path, E_USER_ERROR);
+            #trigger_error('File Error: Failed to remove folder ' . $path, E_USER_ERROR);
         }
     }
     else {
         // delete it
         if (!unlink($path)) {
-            trigger_error('File Error: Failed to remove file ' . $path, E_USER_ERROR);
+            #trigger_error('File Error: Failed to remove file ' . $path, E_USER_ERROR);
         }
     }
 }
 
+function process_category($category, $parent_id = 0) {
+    global $DB;
+
+    $oldid = (int)$category['oldid'];
+    $name = (string)$category['name'];
+
+    $existing_category = \core_course_category::get($oldid, IGNORE_MISSING);
+
+    if ($existing_category->id) {
+        // Update name and parent if necessary
+        $updated_data = [];
+        if ($existing_category->name !== $name) {
+            $updated_data['name'] = $name;
+        }
+        if ($parent_id !== 0 && $existing_category->parent != $parent_id) {
+            $updated_data['parent'] = $parent_id;
+        }
+
+        if (!empty($updated_data)) {
+            // Update category
+            $existing_category->update($updated_data);
+        }
+    } else {
+        // Create categories up to the required ID
+        $max_id = $DB->get_field_sql("SELECT MAX(id) FROM {course_categories}");
+        while ($max_id < $oldid) {
+            $new_category_data = [
+                'name' => "Placeholder " . ($max_id + 1),
+                'parent' => $parent_id,
+                'idnumber' => '',
+                'visible' => 1
+            ];
+
+            // Create placeholder category
+            \core_course_category::create($new_category_data);
+            $max_id = $DB->get_field_sql("SELECT MAX(id) FROM {course_categories}");
+        }
+
+        // Create target category
+        $new_category_data = [
+            'id' => $oldid,
+            'name' => $name,
+            'parent' => $parent_id,
+            'idnumber' => '',
+            'visible' => 1
+        ];
+        $existing_category = \core_course_category::get($oldid, MUST_EXIST, true);
+        $existing_category->update($new_category_data);
+    }
+
+    // Process sub-categories
+    foreach ($category->category as $sub_category) {
+        process_category($sub_category, $oldid);
+    }
+}
+
+$usage = "Import a course from ContentMonster
+
+Usage:
+    # php import_cli.php --courseid=<value> --categoryid=<value> --timestamp=<value> [--source=<value>] [--categories]
+    # php import_cli.php [--help|-h]
+
+Options:
+    -h --help               Print this help.
+    --paramname=<value>     Describe the parameter and the meaning of its values.
+";
+
+list($options, $unrecognised) = cli_get_params([
+    'help' => false,
+    'categories' => false,
+    'courseid' => null,
+    'categoryid' => null,
+    'timestamp' => null,
+    'source' => null,
+], [
+    'h' => 'help',
+    'c' => 'categories'
+]);
+
+if ($unrecognised) {
+    $unrecognised = implode(PHP_EOL . '  ', $unrecognised);
+    cli_error(get_string('cliunknowoption', 'core_admin', $unrecognised));
+}
+
+if ($options['help']) {
+    cli_writeln($usage);
+    exit(2);
+}
+
+$replicationconfig = get_config('local_replication');
+$directory = $replicationconfig->directory;
+
+if (empty($options['timestamp'])) {
+    cli_error('Missing mandatory argument timestamp.', 2);
+}
+
+$timestamp = $options["timestamp"];
+
+if (!empty($options['categories'])) {
+    $path = $directory . DIRECTORY_SEPARATOR . "categories_$timestamp.xml";
+    $xml = simplexml_load_file($path);
+    foreach ($xml->category as $category) {
+        process_category($category);
+    }    
+    removePath($path);
+    echo("OK");
+    exit();
+}
+
+if (empty($options['courseid'])) {
+    cli_error('Missing mandatory argument courseid.', 2);
+}
+
+if (empty($options['categoryid'])) {
+    cli_error('Missing mandatory argument categoryid.', 2);
+}
+
 
 $courseid = $options['courseid'];
 $categoryid = $options["categoryid"];
-$timestamp = $options["timestamp"];
 
 try {
     $course = get_course($courseid);
@@ -114,9 +187,6 @@ if (!$course) {
     }
 }
 
-$replicationconfig = get_config('local_replication');
-$directory = $replicationconfig->directory;
-
 $infile = $directory . DIRECTORY_SEPARATOR . "course_" . $courseid . "_" . $categoryid . "_" . $timestamp . ".mbz";
 
 if (empty($CFG->tempdir)) {
@@ -247,7 +317,11 @@ $course->shortname = $shortname;
 
 $DB->update_record('course', $course);
 
-removePath($path);
+try {
+    removePath($path);
+} catch (\Throwable $th) {
+    cli_writeln("Could not remove temporary files. Please remove them manually.");
+}
 
 cli_writeln("New course ID for '$shortname': $courseid in category {$category->id}");
 cli_writeln("OK");

trigger.php

@@ -2,44 +2,40 @@
 require_once("../../config.php");
 require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 
-$id = $_GET["id"];
+if (! function_exists('str_ends_with')) {
-$course = get_course($id);
+    function str_ends_with(string $haystack, string $needle): bool
+    {
+        $needle_len = strlen($needle);
+        return ($needle_len === 0 || 0 === substr_compare($haystack, $needle, - $needle_len));
+    }
+}
 
 $replicationconfig = get_config('local_replication');
 $directory = $replicationconfig->directory;
 
+if (!str_ends_with($directory, "/")) $directory = $directory . "/";
+
+if (isset($_GET["categories"])) {
+    touch($directory . "categories.mew");
+    echo("Done.");
+    exit();
+}
+
+if (!isset($_GET["id"])) {
+    die("Missing course ID!");
+}
+
+$id = $_GET["id"];
+$course = get_course($id);
+
 $context = context_course::instance($id);
 
 if (!has_capability('local/replication:replicate', $context)) {
     die("User not allowed to trigger replication!");
 }
 
-$bc = new backup_controller(\backup::TYPE_1COURSE, $id, backup::FORMAT_MOODLE,
+if (!touch($directory . $id . "-" . $course->category . ".mew")) {
-    backup::INTERACTIVE_YES, backup::MODE_GENERAL, $USER->id);
+    die("Could not touch $directory$id-{$course->category}.mew");
-
-$tasks = $bc->get_plan()->get_tasks();
-foreach ($tasks as &$task) {
-    if ($task instanceof \backup_root_task) {
-        $setting = $task->get_setting('users');
-        $setting->set_value('0');
-        $setting = $task->get_setting('anonymize');
-        $setting->set_value('1');
-        $setting = $task->get_setting('role_assignments');
-        $setting->set_value('0');
-        $setting = $task->get_setting('filters');
-        $setting->set_value('0');
-        $setting = $task->get_setting('comments');
-        $setting->set_value('0');
-        $setting = $task->get_setting('logs');
-        $setting->set_value('0');
-        $setting = $task->get_setting('grade_histories');
-        $setting->set_value('0');
-    } 
 }
 
-$filename = $directory . '/course_' . $id . "_" . $course->category . "_" . date('U') . '.mbz';
+echo("Done.");
-
-$bc->set_status(backup::STATUS_AWAITING);
-$bc->execute_plan();
-
-echo('Course is now getting replicated. <a href="javascript:history.back();">Back to Course Administration</a>');

trigger_all.php

@@ -0,0 +1,33 @@
+<?php
+require_once("../../config.php");
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+
+if (! function_exists('str_ends_with')) {
+    function str_ends_with(string $haystack, string $needle): bool
+    {
+        $needle_len = strlen($needle);
+        return ($needle_len === 0 || 0 === substr_compare($haystack, $needle, - $needle_len));
+    }
+}
+
+
+$context = context_system::instance();
+
+if (!has_capability('local/replication:replicate', $context)) {
+    die("User not allowed to trigger replication!");
+}
+
+$courses = get_courses();
+
+$replicationconfig = get_config('local_replication');
+$directory = $replicationconfig->directory;
+
+if (!str_ends_with($directory, "/")) $directory = $directory . "/";
+
+foreach ($courses as $course) {
+    if (!touch($directory . $course->id . "-" . $course->category . ".mew")) {
+        die("Could not touch $directory$id-{$course->category}.mew");
+    }
+}
+
+echo("Done.");

unenroll.php

@@ -0,0 +1,75 @@
+<?php
+
+define('CLI_SCRIPT', true);
+require(__DIR__ . '/../../config.php');
+require_once($CFG->libdir . '/clilib.php');
+require_once($CFG->libdir . '/enrollib.php');
+require_once($CFG->libdir . '/completionlib.php');
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+
+$usage = "Usage:
+    unenroll.php --username=<username> --shortname=<shortname>
+
+Options:
+    --username    The username of the user to unenroll.
+    --shortname   The shortname of the course to unenroll the user from.
+
+";
+
+list($options, $unrecognized) = cli_get_params(
+    [
+        'username' => null,
+        'shortname' => null,
+        'help' => false,
+    ],
+    ['h' => 'help']
+);
+
+if ($options['help'] || !$options['username'] || !$options['shortname']) {
+    echo $usage;
+    exit(0);
+}
+
+$username = $options['username'];
+$shortname = $options['shortname'];
+
+try {
+    $user = $DB->get_record('user', array('username' => $username), '*', MUST_EXIST);
+} catch (Exception $ex) {
+    cli_error('User ' . $username . ' does not exist.');
+}
+
+try {
+    $course = $DB->get_record('course', array('shortname' => $shortname), '*', MUST_EXIST);
+} catch (Exception $ex) {
+    cli_error('Course ' . $shortname . ' does not exist.');
+}
+
+$context = context_course::instance($course->id);
+
+$courseid = $course->id;
+$userid = $user->id;
+
+$context = context_course::instance($courseid);
+$enrolinstances = enrol_get_instances($courseid, true);
+$enrolplugin = enrol_get_plugin('manual');
+
+$manualinstance = null;
+foreach ($enrolinstances as $instance) {
+    if ($instance->enrol == 'manual') {
+        $manualinstance = $instance;
+        break;
+    }
+}
+if ($manualinstance === null) {
+    cli_error('No manual enrollment instance found for the course.');
+}
+
+$enrolplugin->unenrol_user($manualinstance, $userid);
+
+$DB->delete_records('course_completions', ['userid' => $userid, 'course' => $courseid]);
+$DB->delete_records('course_completion_crit_compl', ['userid' => $userid, 'course' => $courseid]);
+
+backup_plan_builder::delete_course_userdata($userid, $courseid);
+
+cli_writeln("User '{$username}' has been unenrolled from course '{$shortname}' and their completion data has been removed.");