{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":752109847,"defaultBranch":"gh-task-runner","name":"lm-evaluation-harness","ownerLogin":"RWKV","currentUserCanPush":false,"isFork":true,"isEmpty":false,"createdAt":"2024-02-03T03:52:04.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/132652788?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1710691791.0","currentOid":""},"activityList":{"items":[{"before":"3a465be24d59ab60037fda4193074fa4b673f824","after":"8630d245514411f83a176fb32b692fa5bfc1347d","ref":"refs/heads/gh-task-runner","pushedAt":"2024-05-08T21:27:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Update action.yaml","shortMessageHtmlLink":"Update action.yaml"}},{"before":"f0cec72abe8bb52ceb2aa47237dfa7110e04cc76","after":"3a465be24d59ab60037fda4193074fa4b673f824","ref":"refs/heads/gh-task-runner","pushedAt":"2024-04-16T13:38:36.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Clean up stuff from old task","shortMessageHtmlLink":"Clean up stuff from old task"}},{"before":"b208b9fb6d4857dd2967c7a6f3511d2e481a8d2f","after":"f0cec72abe8bb52ceb2aa47237dfa7110e04cc76","ref":"refs/heads/gh-task-runner","pushedAt":"2024-04-16T13:33:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Fix typo","shortMessageHtmlLink":"Fix typo"}},{"before":"1ee41f73a222c21bffbf3343e450fd4833804f78","after":"b208b9fb6d4857dd2967c7a6f3511d2e481a8d2f","ref":"refs/heads/gh-task-runner","pushedAt":"2024-04-16T13:32:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Update gh-task-runner-SQLLM.yml","shortMessageHtmlLink":"Update gh-task-runner-SQLLM.yml"}},{"before":"bff08d1602e9a76bf402049540ca38ba5682b61a","after":"1ee41f73a222c21bffbf3343e450fd4833804f78","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-22T22:31:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"SmerkyG","name":null,"path":"/SmerkyG","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/8826350?s=80&v=4"},"commit":{"message":"install latest version of transformers","shortMessageHtmlLink":"install latest version of transformers"}},{"before":"e1b8476650ea59d86a343818aa0dc454d7645539","after":"bff08d1602e9a76bf402049540ca38ba5682b61a","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-17T16:12:48.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Fix matrix and rename file","shortMessageHtmlLink":"Fix matrix and rename file"}},{"before":"b923a38b8837f9e217752f8ada12566b55e8b70d","after":"e1b8476650ea59d86a343818aa0dc454d7645539","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-17T16:09:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Automate sqllm evals","shortMessageHtmlLink":"Automate sqllm evals"}},{"before":"318e6b82184b2a91a95bdb7ebe2ace1bf2e610bf","after":"b923a38b8837f9e217752f8ada12566b55e8b70d","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-11T02:19:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"eval tweak","shortMessageHtmlLink":"eval tweak"}},{"before":"173e67b57e60e14922845057395bbb0a8c44b264","after":"318e6b82184b2a91a95bdb7ebe2ace1bf2e610bf","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-10T21:12:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"fixing the HF cache dir path","shortMessageHtmlLink":"fixing the HF cache dir path"}},{"before":"8281e96fdf1d5d28a8fe9b34b311f55a87e62807","after":"173e67b57e60e14922845057395bbb0a8c44b264","ref":"refs/heads/gh-task-runner","pushedAt":"2024-03-10T20:59:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"fixing subpath issue with .pth file evals","shortMessageHtmlLink":"fixing subpath issue with .pth file evals"}},{"before":"e53d1c509efabcfe2f9d1eaa515e748ee88b50d7","after":"8281e96fdf1d5d28a8fe9b34b311f55a87e62807","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-28T09:11:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Addded sentencepiece to deps\n\nFor sealion model test","shortMessageHtmlLink":"Addded sentencepiece to deps"}},{"before":"71d574c475804c908d4ae6483c83fe3ccb8b2cb9","after":"e53d1c509efabcfe2f9d1eaa515e748ee88b50d7","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-27T07:30:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"dropping trivaqa","shortMessageHtmlLink":"dropping trivaqa"}},{"before":"c2f0db9db7d63d255701648804df9471f7a7a8f1","after":null,"ref":"refs/heads/pile-fix","pushedAt":"2024-02-26T18:54:41.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"}},{"before":null,"after":"c2f0db9db7d63d255701648804df9471f7a7a8f1","ref":"refs/heads/gh-task-runner-pile","pushedAt":"2024-02-26T18:45:40.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Move pile.py to correct subfolder","shortMessageHtmlLink":"Move pile.py to correct subfolder"}},{"before":"ea10da61a38d0cc7900d606bef1f900ea5c06049","after":"71d574c475804c908d4ae6483c83fe3ccb8b2cb9","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-26T18:30:18.000Z","pushType":"pr_merge","commitsCount":2,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Merge pull request #4 from RWKV/revert-2-main\n\nRevert \"Pull in latest lm-eval updates into the our gh-task-runner branch\"","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2154839695\" data-permission-text=\"Title is private\" data-url=\"https://github.com/RWKV/lm-evaluation-harness/issues/4\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/RWKV/lm-evaluation-harness/pull/4/hovercard\" href=\"https://github.com/RWKV/lm-evaluation-harness/pull/4\">#4</a> from RWKV/revert-2-main"}},{"before":null,"after":"6690829f0122e85b740df3e0cca50ae3ae17ac96","ref":"refs/heads/revert-2-main","pushedAt":"2024-02-26T18:30:03.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Revert \"Pull in latest lm-eval updates into the our gh-task-runner branch\"","shortMessageHtmlLink":"Revert \"Pull in latest lm-eval updates into the our gh-task-runner br…"}},{"before":"f9d79a79598f63833a2c8f001c88f9c0418c95d9","after":"c2f0db9db7d63d255701648804df9471f7a7a8f1","ref":"refs/heads/pile-fix","pushedAt":"2024-02-26T17:32:41.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Move pile.py to correct subfolder","shortMessageHtmlLink":"Move pile.py to correct subfolder"}},{"before":"cc9f6c175a03dafec1c5b5731b45edc5e2042589","after":"f9d79a79598f63833a2c8f001c88f9c0418c95d9","ref":"refs/heads/pile-fix","pushedAt":"2024-02-26T17:31:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Add pile dataset reference","shortMessageHtmlLink":"Add pile dataset reference"}},{"before":"ea10da61a38d0cc7900d606bef1f900ea5c06049","after":"cc9f6c175a03dafec1c5b5731b45edc5e2042589","ref":"refs/heads/pile-fix","pushedAt":"2024-02-26T17:29:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Correct Pile split","shortMessageHtmlLink":"Correct Pile split"}},{"before":null,"after":"ea10da61a38d0cc7900d606bef1f900ea5c06049","ref":"refs/heads/pile-fix","pushedAt":"2024-02-26T17:27:58.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"redbrain","name":"redbrain","path":"/redbrain","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/42494361?s=80&v=4"},"commit":{"message":"Add GH Task Runner workflow for focused suite","shortMessageHtmlLink":"Add GH Task Runner workflow for focused suite"}},{"before":"f8bc085e28ad04eaf018d70614b00aab0c4709a4","after":"ea10da61a38d0cc7900d606bef1f900ea5c06049","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-26T09:20:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Add GH Task Runner workflow for focused suite","shortMessageHtmlLink":"Add GH Task Runner workflow for focused suite"}},{"before":"5e02eea6341a1bff938ea8215b06bcd218e6168f","after":"f8bc085e28ad04eaf018d70614b00aab0c4709a4","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-26T03:19:19.000Z","pushType":"pr_merge","commitsCount":34,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Merge pull request #2 from RWKV/main\n\nPull in latest lm-eval updates into the our gh-task-runner branch","shortMessageHtmlLink":"Merge pull request <a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2153010060\" data-permission-text=\"Title is private\" data-url=\"https://github.com/RWKV/lm-evaluation-harness/issues/2\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/RWKV/lm-evaluation-harness/pull/2/hovercard\" href=\"https://github.com/RWKV/lm-evaluation-harness/pull/2\">#2</a> from RWKV/main"}},{"before":"f78e2da45f034a23b1b13cde3235105b0f55d830","after":"a95552e6dd76e517ab45770f78756a6dff96820a","ref":"refs/heads/main","pushedAt":"2024-02-26T03:17:08.000Z","pushType":"push","commitsCount":162,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Merge branch 'gh-task-runner' into main","shortMessageHtmlLink":"Merge branch 'gh-task-runner' into main"}},{"before":"7411947112117e0339fe207fb620a70bcec22690","after":"f78e2da45f034a23b1b13cde3235105b0f55d830","ref":"refs/heads/main","pushedAt":"2024-02-26T00:10:23.000Z","pushType":"push","commitsCount":32,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Add environment and transformers version logging in results dump (#1464)\n\n* Save git_hash to results even if git is not available to call as subprocess\r\n\r\n* Store more info about environment and transformers version in results to help researchers track inconsistencies\r\n\r\n* moved added logging to logging_utils\r\n\r\n* moved get_git_commit_hash to logging_utils.py\r\n\r\n* moved add_env_info inside evaluator","shortMessageHtmlLink":"Add environment and transformers version logging in results dump (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2151457922\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/1464\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/1464/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/1464\">Ele…</a>"}},{"before":"f7ea5c52a089e8c9288e256046861df0ee38aa9c","after":"5e02eea6341a1bff938ea8215b06bcd218e6168f","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T23:25:19.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"removing the 40GB vram option skew","shortMessageHtmlLink":"removing the 40GB vram option skew"}},{"before":"808b8a73fb0dbfbd728efb0492559449b2301c51","after":"f7ea5c52a089e8c9288e256046861df0ee38aa9c","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T22:36:29.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"reverting the need for git clone reset of the HF conversion - the issue was isolated to folder name including a `.`","shortMessageHtmlLink":"reverting the need for git clone reset of the HF conversion - the iss…"}},{"before":"54b4e5a38415cb30a3e9658a3b80b00dc82947f1","after":"808b8a73fb0dbfbd728efb0492559449b2301c51","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T22:11:29.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Bug isolating","shortMessageHtmlLink":"Bug isolating"}},{"before":"21ea2bea631c97bc100a3e9ea16c0abbcd76969e","after":"54b4e5a38415cb30a3e9658a3b80b00dc82947f1","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T22:06:40.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Add support for custom models in task runner composite action","shortMessageHtmlLink":"Add support for custom models in task runner composite action"}},{"before":"0adf07bb44ebaf298b257060fae0c945e9497aee","after":"21ea2bea631c97bc100a3e9ea16c0abbcd76969e","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T21:33:35.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Merge branch 'gh-task-runner' of https://github.com/RWKV/lm-evaluation-harness into gh-task-runner","shortMessageHtmlLink":"Merge branch 'gh-task-runner' of <a href=\"https://github.com/RWKV/lm-evaluation-harness\">https://github.com/RWKV/lm-evaluatio…</a>"}},{"before":"7b3865f5f08924b8e0fc85e74f4f1ab378cf6693","after":"0adf07bb44ebaf298b257060fae0c945e9497aee","ref":"refs/heads/gh-task-runner","pushedAt":"2024-02-25T19:14:23.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"PicoCreator","name":"Eugene Cheah","path":"/PicoCreator","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/17175484?s=80&v=4"},"commit":{"message":"Merge branch 'gh-task-runner' of https://github.com/RWKV/lm-evaluation-harness into gh-task-runner","shortMessageHtmlLink":"Merge branch 'gh-task-runner' of <a href=\"https://github.com/RWKV/lm-evaluation-harness\">https://github.com/RWKV/lm-evaluatio…</a>"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAERQ5NXQA","startCursor":null,"endCursor":null}},"title":"Activity · RWKV/lm-evaluation-harness"}