From 0a7f5930b7011a87f5d18ee4c79ed1865c52fe59 Mon Sep 17 00:00:00 2001
From: Atlas Unified <AtlasUnified@users.noreply.huggingface.co>
Date: Thu, 29 Jun 2023 22:36:04 +0000
Subject: [PATCH] Upload openorca.py

---
 openorca.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 openorca.py

diff --git a/openorca.py b/openorca.py
new file mode 100644
index 0000000..29b2cc7
--- /dev/null
+++ b/openorca.py
@@ -0,0 +1,144 @@
+import evaluate
+from evaluate.evaluation_suite import SubTask
+
+class MyEvaluationSuite(evaluate.EvaluationSuite):
+
+    def __init__(self, name):
+        super().__init__(name)
+        self.suite = [
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="001-cot",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0.0,
+                        "your-label-1": 1.0
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="002-flan",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="003-flan-1m",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="004-flan1m-aug-shuf",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="005-flan-5m",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="006-flan-chatgpt",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="007-gpt4_100k",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="008-niv",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="OpenOrca",
+                subset="009-t0",
+                split="your-split",
+                args_for_task={
+                    "metric": "your-metric",
+                    "input_column": "your-input-column",
+                    "label_column": "your-label-column",
+                    "label_mapping": {
+                        "your-label-0": 0,
+                        "your-label-1": 1
+                    }
+                }
+            )
+        ]
\ No newline at end of file