Convert activation functions to numpower

.github/workflows/ci.yml

@@ -8,14 +8,8 @@ jobs:
     runs-on: ${{ matrix.operating-system }}
     strategy:
       matrix:
-<<<<<<< HEAD
         operating-system: [windows-latest, ubuntu-latest, macos-latest]
         php-versions: ['8.4']
-=======
-        operating-system: [ubuntu-latest, macos-latest]
-        php-versions: ['8.0', '8.1', '8.2']
->>>>>>> master
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.gitignore

@@ -13,3 +13,12 @@ pyvenv.cfg
 .idea
 .vscode
 .vs
+
+# Cache files
+/runtime/
+
+# Docker related files
+Dockerfile
+/docker/
+docker-compose.yml
+Makefile

.php-cs-fixer.dist.php

@@ -3,115 +3,128 @@
 use PhpCsFixer\Finder;
 use PhpCsFixer\Config;
 
-$finder = Finder::create()->in(__DIR__)
-    ->exclude('docs');
+$finder = Finder::create()
+    ->exclude([
+        __DIR__ . '/docs/',
+        __DIR__ . '/vendor/',
+    ])
+    ->in(__DIR__)
+    ->append([
+        __FILE__,
+    ]);
 
 $config = new Config();
+$config
+    ->setCacheFile(__DIR__ . '/runtime/.php-cs-fixer.cache')
+    ->setRules(
+        [
+            '@PSR2' => true,
+            '@PHP84Migration' => true,
+            'align_multiline_comment' => true,
+            'array_syntax' => ['syntax' => 'short'],
+            'backtick_to_shell_exec' => true,
+            'binary_operator_spaces' => true,
+            'blank_lines_before_namespace' => true,
+            'blank_line_after_namespace' => true,
+            'blank_line_after_opening_tag' => true,
+            'blank_line_before_statement' => [
+                'statements' => [
+                    'break', 'case', 'continue', 'declare', 'default', 'do', 'for',
+                    'if', 'foreach', 'return', 'switch', 'try', 'while',
+                ],
+            ],
+            'cast_spaces' => ['space' => 'single'],
+            'class_attributes_separation' => true,
+            'combine_consecutive_issets' => true,
+            'combine_consecutive_unsets' => true,
+            'compact_nullable_type_declaration' => true,
+            'concat_space' => ['spacing' => 'one'],
+            'fully_qualified_strict_types' => true,
+            'increment_style' => ['style' => 'pre'],
+            'linebreak_after_opening_tag' => true,
+            'list_syntax' => ['syntax' => 'short'],
+            'lowercase_cast' => true,
+            'lowercase_static_reference' => true,
+            'magic_constant_casing' => true,
+            'magic_method_casing' => true,
+            'multiline_comment_opening_closing' => true,
+            'multiline_whitespace_before_semicolons' => [
+                'strategy' => 'no_multi_line',
+            ],
+            'native_function_casing' => true,
+            'native_type_declaration_casing' => true,
+            'new_with_parentheses' => true,
+            'no_alternative_syntax' => true,
+            'no_blank_lines_after_class_opening' => true,
+            'no_blank_lines_after_phpdoc' => true,
+            'no_empty_statement' => true,
+            'no_extra_blank_lines' => true,
+            'no_leading_import_slash' => true,
+            'no_leading_namespace_whitespace' => true,
+            'no_mixed_echo_print' => ['use' => 'echo'],
+            'no_null_property_initialization' => true,
+            'no_short_bool_cast' => true,
+            'no_singleline_whitespace_before_semicolons' => true,
+            'no_spaces_around_offset' => true,
+            'no_superfluous_phpdoc_tags' => false,
+            'no_superfluous_elseif' => true,
+            'no_trailing_comma_in_singleline' => true,
+            'no_unneeded_control_parentheses' => true,
+            'no_unneeded_braces' => true,
+            'no_unset_cast' => true,
+            'no_unused_imports' => true,
+            'no_useless_else' => true,
+            'no_useless_return' => true,
+            'no_whitespace_before_comma_in_array' => true,
+            'no_whitespace_in_blank_line' => true,
+            'normalize_index_brace' => true,
+            'nullable_type_declaration_for_default_null_value' => true,
+            'object_operator_without_whitespace' => true,
+            'ordered_class_elements' => [
+                'order' => [
+                    'use_trait', 'constant_public', 'constant_protected',
+                    'constant_private', 'property_public_static', 'property_protected_static',
+                    'property_private_static', 'property_public', 'property_protected',
+                    'property_private', 'method_public_static', 'method_protected_static',
+                    'method_private_static', 'construct', 'destruct', 'phpunit',
+                    'method_public', 'method_protected', 'method_private', 'magic',
+                ],
+                'sort_algorithm' => 'none',
+            ],
+            'php_unit_fqcn_annotation' => true,
+            'php_unit_method_casing' => ['case' => 'camel_case'],
+            'phpdoc_add_missing_param_annotation' => ['only_untyped' => false],
+            'phpdoc_align' => ['align' => 'left'],
+            'phpdoc_line_span' => [
+                'const' => 'multi',
+                'method' => 'multi',
+                'property' => 'multi',
+            ],
+            'phpdoc_no_access' => true,
+            'phpdoc_no_empty_return' => true,
+            'phpdoc_no_useless_inheritdoc' => true,
+            'phpdoc_order' => true,
+            'phpdoc_scalar' => true,
+            'phpdoc_single_line_var_spacing' => true,
+            'phpdoc_to_comment' => false,
+            'phpdoc_trim' => true,
+            'phpdoc_trim_consecutive_blank_line_separation' => true,
+            'phpdoc_var_without_name' => true,
+            'protected_to_private' => true,
+            'return_assignment' => false,
+            'return_type_declaration' => ['space_before' => 'one'],
+            'semicolon_after_instruction' => true,
+            'short_scalar_cast' => true,
+            'simplified_null_return' => true,
+            'single_quote' => true,
+            'single_line_comment_style' => true,
+            'ternary_operator_spaces' => true,
+            'ternary_to_null_coalescing' => true,
+            'type_declaration_spaces' => true,
+            'trim_array_spaces' => true,
+            'unary_operator_spaces' => true,
+            'whitespace_after_comma_in_array' => true,
+        ]
+    )->setFinder($finder);
 
-return $config->setRules([
-    '@PSR2' => true,
-    'align_multiline_comment' => true,
-    'array_syntax' => ['syntax' => 'short'],
-    'backtick_to_shell_exec' => true,
-    'binary_operator_spaces' => true,
-    'blank_lines_before_namespace' => true,
-    'blank_line_after_namespace' => true,
-    'blank_line_after_opening_tag' => true,
-    'blank_line_before_statement' => [
-        'statements' => [
-            'break', 'case', 'continue', 'declare', 'default', 'do', 'for',
-            'if', 'foreach', 'return', 'switch', 'try', 'while',
-        ],
-    ],
-    'cast_spaces' => ['space' => 'single'],
-    'class_attributes_separation' => true,
-    'combine_consecutive_issets' => true,
-    'combine_consecutive_unsets' => true,
-    'compact_nullable_type_declaration' => true,
-    'concat_space' => ['spacing' => 'one'],
-    'fully_qualified_strict_types' => true,
-    'increment_style' => ['style' => 'pre'],
-    'linebreak_after_opening_tag' => true,
-    'list_syntax' => ['syntax' => 'short'],
-    'lowercase_cast' => true,
-    'lowercase_static_reference' => true,
-    'magic_constant_casing' => true,
-    'magic_method_casing' => true,
-    'multiline_comment_opening_closing' => true,
-    'multiline_whitespace_before_semicolons' => [
-        'strategy' => 'no_multi_line',
-    ],
-    'native_function_casing' => true,
-    'native_type_declaration_casing' => true,
-    'new_with_parentheses' => true,
-    'no_alternative_syntax' => true,
-    'no_blank_lines_after_class_opening' => true,
-    'no_blank_lines_after_phpdoc' => true,
-    'no_empty_statement' => true,
-    'no_extra_blank_lines' => true,
-    'no_leading_import_slash' => true,
-    'no_leading_namespace_whitespace' => true,
-    'no_mixed_echo_print' => ['use' => 'echo'],
-    'no_null_property_initialization' => true,
-    'no_short_bool_cast' => true,
-    'no_singleline_whitespace_before_semicolons' => true,
-    'no_spaces_around_offset' => true,
-    'no_superfluous_phpdoc_tags' => false,
-    'no_superfluous_elseif' => true,
-    'no_trailing_comma_in_singleline' => true,
-    'no_unneeded_control_parentheses' => true,
-    'no_unneeded_braces' => true,
-    'no_unset_cast' => true,
-    'no_unused_imports' => true,
-    'no_useless_else' => true,
-    'no_useless_return' => true,
-    'no_whitespace_before_comma_in_array' => true,
-    'no_whitespace_in_blank_line' => true,
-    'normalize_index_brace' => true,
-    'nullable_type_declaration_for_default_null_value' => true,
-    'object_operator_without_whitespace' => true,
-    'ordered_class_elements' => [
-        'order' => [
-            'use_trait', 'constant_public', 'constant_protected',
-            'constant_private', 'property_public_static', 'property_protected_static',
-            'property_private_static', 'property_public', 'property_protected',
-            'property_private', 'method_public_static', 'method_protected_static',
-            'method_private_static', 'construct', 'destruct', 'phpunit',
-            'method_public', 'method_protected', 'method_private', 'magic',
-        ],
-        'sort_algorithm' => 'none',
-    ],
-    'php_unit_fqcn_annotation' => true,
-    'php_unit_method_casing' => ['case' => 'camel_case'],
-    'phpdoc_add_missing_param_annotation' => ['only_untyped' => false],
-    'phpdoc_align' => ['align' => 'left'],
-    'phpdoc_line_span' => [
-        'const' => 'multi',
-        'method' => 'multi',
-        'property' => 'multi',
-    ],
-    'phpdoc_no_access' => true,
-    'phpdoc_no_empty_return' => true,
-    'phpdoc_no_useless_inheritdoc' => true,
-    'phpdoc_order' => true,
-    'phpdoc_scalar' => true,
-    'phpdoc_single_line_var_spacing' => true,
-    'phpdoc_to_comment' => false,
-    'phpdoc_trim' => true,
-    'phpdoc_trim_consecutive_blank_line_separation' => true,
-    'phpdoc_var_without_name' => true,
-    'protected_to_private' => true,
-    'return_assignment' => false,
-    'return_type_declaration' => ['space_before' => 'one'],
-    'semicolon_after_instruction' => true,
-    'short_scalar_cast' => true,
-    'simplified_null_return' => true,
-    'single_quote' => true,
-    'single_line_comment_style' => true,
-    'ternary_operator_spaces' => true,
-    'ternary_to_null_coalescing' => true,
-    'type_declaration_spaces' => true,
-    'trim_array_spaces' => true,
-    'unary_operator_spaces' => true,
-    'whitespace_after_comma_in_array' => true,
-])->setFinder($finder);
+return $config;

.phplint.yml

@@ -0,0 +1,8 @@
+path: ./
+jobs: 10
+cache-dir: runtime/.phplint.cache/
+extensions:
+    - php
+exclude:
+    - vendor/
+    - runtime/

composer.json

@@ -44,6 +44,7 @@
     "require-dev": {
         "friendsofphp/php-cs-fixer": "^3.73",
         "phpbench/phpbench": "^1.0",
+        "overtrue/phplint": "^9.6.2",
         "phpstan/extension-installer": "^1.0",
         "phpstan/phpstan": "^2.0",
         "phpstan/phpstan-phpunit": "^2.0",
@@ -89,6 +90,7 @@
             "@putenv PHP_CS_FIXER_IGNORE_ENV=1",
             "php-cs-fixer fix --config=.php-cs-fixer.dist.php"
         ],
+        "phplint": "phplint",
         "test": "phpunit"
     },
     "config": {

docs/neural-network/activation-functions/elu.md

@@ -1,23 +1,32 @@
-<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/ELU.php">[source]</a></span>
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/ELU/ELU.php">[source]</a></span>
 
 # ELU
 *Exponential Linear Units* are a type of rectifier that soften the transition from non-activated to activated using the exponential function. As such, ELU produces smoother gradients than the piecewise linear [ReLU](relu.md) function.
 
 $$
-{\displaystyle ELU = {\begin{cases}\alpha \left(e^{x}-1\right)&{\text{if }}x\leq 0\\x&{\text{if }}x>0\end{cases}}}
+\text{ELU}(x) =
+\begin{cases}
+\alpha \left(e^{x}-1\right) & \text{if } x \leq 0 \\
+x & \text{if } x > 0
+\end{cases}
 $$
 
 ## Parameters
 | # | Name | Default | Type | Description |
 |---|---|---|---|---|
 | 1 | alpha | 1.0 | float | The value at which leakage will begin to saturate. Ex. alpha = 1.0 means that the output will never be less than -1.0 when inactivated. |
 
+## Plots
+<img src="../../images/activation-functions/elu.png" alt="ELU Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/elu-derivative.png" alt="ELU Derivative" width="500" height="auto">
+
 ## Example
 ```php
-use Rubix\ML\NeuralNet\ActivationFunctions\ELU;
+use Rubix\ML\NeuralNet\ActivationFunctions\ELU\ELU;
 
 $activationFunction = new ELU(2.5);
 ```
 
 ## References
-[^1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.
+[1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.

docs/neural-network/activation-functions/gelu.md

@@ -1,14 +1,23 @@
-<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/GELU.php">[source]</a></span>
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/GELU/GELU.php">[source]</a></span>
 
 # GELU
 Gaussian Error Linear Units (GELUs) are rectifiers that are gated by the magnitude of their input rather than the sign of their input as with ReLU variants. Their output can be interpreted as the expected value of a neuron with random dropout regularization applied.
 
+$$
+\text{GELU}(x) = 0.5 \cdot x \left(1 + \tanh\left(\sqrt{\frac{2}{\pi}} \left(x + 0.044715 \cdot x^3\right)\right)\right)
+$$
+
 ## Parameters
 This activation function does not have any parameters.
 
+## Plots
+<img src="../../images/activation-functions/gelu.png" alt="GELU Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/gelu-derivative.png" alt="GELU Derivative" width="500" height="auto">
+
 ## Example
 ```php
-use Rubix\ML\NeuralNet\ActivationFunctions\GELU;
+use Rubix\ML\NeuralNet\ActivationFunctions\GELU\GELU;
 
 $activationFunction = new GELU();
 ```

docs/neural-network/activation-functions/hard-sigmoid.md

@@ -0,0 +1,26 @@
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php">[source]</a></span>
+
+# Hard Sigmoid
+A piecewise linear approximation of the sigmoid function that is computationally more efficient. The Hard Sigmoid function has an output value between 0 and 1, making it useful for binary classification problems.
+
+$$
+\text{HardSigmoid}(x) = \max\left(0,\min\left(1, 0.2x + 0.5\right)\right)
+$$
+
+## Parameters
+This activation function does not have any parameters.
+
+## Plots
+<img src="../../images/activation-functions/hard-sigmoid.png" alt="Hard Sigmoid Function" width="500" height="auto">
+
+<img src="../../images/activation-functions/hard-sigmoid-derivative.png" alt="Hard Sigmoid Derivative" width="500" height="auto">
+
+## Example
+```php
+use Rubix\ML\NeuralNet\ActivationFunctions\HardSigmoid\HardSigmoid;
+
+$activationFunction = new HardSigmoid();
+```
+
+## References
+[1]: https://en.wikipedia.org/wiki/Hard_sigmoid