LSio: improve handling time & memory limits and allow before_import hook to set them

src/includes/class/class.LSio.php

@@ -227,10 +227,6 @@ class LSio extends LSlog_staticLoggerClass {
     $objectsInError = array();
     self :: log_trace("import(): objects data=".varDump($objectsData));
 
-    // Reset & increase time limit: allow one seconds by object to handle,
-    // with a minimum of 30 seconds
-    set_time_limit((count($objectsData)>30?count($objectsData):30));
-
     // Trigger before_import event
     if (
       !$ioFormat -> fireEvent(
@@ -248,8 +244,79 @@ class LSio extends LSlog_staticLoggerClass {
       return $return;
     }
 
+    // Increase time limit: allow at least one seconds by object to handle,
+    // with a minimum of 30 seconds
+    $time_limit = intval(ini_get('max_execution_time'));
+    $new_time_limit = (count($objectsData)>30?count($objectsData):30);
+    if ($time_limit == 0) {
+      self :: log_debug("import(): time limit = 0 (no limit)");
+    }
+    else if ($new_time_limit > $time_limit) {
+      self :: log_debug(
+        sprintf(
+          "import(): increase time limit to %s seconds (current: %s seconds)",
+          $new_time_limit, $time_limit
+        )
+      );
+      if (set_time_limit($new_time_limit) === false) {
+        self :: log_warning(
+          sprintf(
+            "import(): fail to increase time limit to %s seconds (current: %s seconds)",
+            $new_time_limit, $time_limit
+          )
+        );
+      }
+      else {
+        $time_limit = $new_time_limit;
+      }
+    }
+    else {
+      self :: log_debug("import(): time limit = $time_limit seconds");
+    }
+
+    // Increase memory limit: allow at least 10M by object to handle
+    $mem_limit = php_ini_unit_to_bytes(ini_get('memory_limit'));
+    $new_mem_limit = 10485760 * count($objectsData);
+    if ($mem_limit == -1) {
+      self :: log_debug("import(): memory limit = -1 (no limit)");
+    }
+    else if ($new_mem_limit > $mem_limit) {
+      self :: log_debug(
+        sprintf(
+          "import(): increase memory limit to %s (current: %s)",
+          format_size($new_mem_limit), format_size($mem_limit)
+        )
+      );
+      if (ini_set('memory_limit', $new_mem_limit."B") === false) {
+        self :: log_warning(
+          sprintf(
+            "import(): fail to increase memory limit to %s (current: %s)",
+            format_size($new_mem_limit), format_size($mem_limit)
+          )
+        );
+      }
+      else {
+        $mem_limit = $new_mem_limit;
+      }
+    }
+    else {
+      self :: log_debug("import(): memory limit = ".format_size($mem_limit));
+    }
+
     // Browse inputed objects
-    foreach($objectsData as $objData) {
+    foreach($objectsData as $idx => $objData) {
+      // Force execution of PHP garbage collector on each object
+      gc_collect_cycles();
+      self :: log_debug(
+        sprintf(
+          'import() - #%s: memory usage = %s (max = %s) on %s',
+          $idx,
+          format_size(memory_get_usage()),
+          format_size(memory_get_peak_usage()),
+          $mem_limit==-1?'no limit':format_size($mem_limit)
+        )
+      );
+
       $globalErrors = array();
       // Instanciate an LSobject
       $object = new $LSobject();
@@ -350,6 +417,17 @@ class LSio extends LSlog_staticLoggerClass {
     $return['errors'] = $objectsInError;
     $return['success'] = empty($objectsInError);
 
+    // Force execution of PHP garbage collector after import
+    gc_collect_cycles();
+    self :: log_debug(
+      sprintf(
+        'import(): memory usage after import = %s (max = %s) on %s',
+        format_size(memory_get_usage()),
+        format_size(memory_get_peak_usage()),
+        $mem_limit==-1?'no limit':format_size($mem_limit)
+      )
+    );
+
     // Trigger after_import event
     if (
       !$ioFormat -> fireEvent(

src/includes/functions.php

@@ -841,3 +841,51 @@ function generate_uuid() {
     mt_rand( 0, 0xffff ), mt_rand( 0, 0xffff ), mt_rand( 0, 0xffff )
   );
 }
+
+/**
+ * Format a number
+ * @param float|int $number
+ * @return string
+ */
+function format_number($number) {
+  if ((int)$number == $number) return strval($number);
+  return number_format($number, 2, ',', '.');
+}
+
+/**
+ * Format a size (in bytes)
+ * @param int  $value
+ * @param boolean|string $unit   Unit of the provided value (optional, default=bytes)
+ * @return string|false
+ */
+function format_size($value, $unit=false) {
+  $units = array(
+    ___('TB') => 1099511627776,
+    ___('GB') => 1073741824,
+    ___('MB') => 1048576,
+    ___('KB') => 1024,
+    ___('B') => 1,
+  );
+  if (!$unit) $unit = 'B';
+  if (!array_key_exists($unit, $units)) return false;
+  $value = $value * $units[$unit];
+  foreach ($units as $unit => $factor) {
+    if ($value >= $factor)
+      return format_number($value / $factor)._($unit);
+  }
+  // 0 ?
+  return strval($value);
+}
+
+/**
+ * Convert PHP ini value with unit as bytes
+ * @param string $value
+ * @return int
+ */
+function php_ini_unit_to_bytes($value) {
+  if ($value == "-1") return -1;
+  return (int)preg_replace_callback('/(\-?\d+)(.?)/', function ($m) {
+    $factor = $m[2]?strpos('BKMG', $m[2]):null;
+    return $factor?intval($m[1]) * pow(1024, $factor):$m[0];
+  }, strtoupper($value));
+}