From 87dc034b590723a7ebf354df576a13690d9664cc Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 31 May 2023 10:57:53 +0200 Subject: [PATCH] feat(server): add retry on download (#384) --- .../test_flash_falcon_load.json | 562 +++++++++--------- .../text_generation_server/models/flash_rw.py | 4 +- server/text_generation_server/models/rw.py | 2 +- server/text_generation_server/utils/hub.py | 39 +- 4 files changed, 309 insertions(+), 298 deletions(-) diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json index 0fb1be756..90a35eb7e 100644 --- a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json +++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json @@ -12,22 +12,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5039062, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -37,27 +37,27 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.04837036, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -3.9960938, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07525635, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006790161, "text": "es" }, { @@ -67,82 +67,82 @@ }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.109375, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.09375, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1845703, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77734375, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3828125, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.0044403076, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9667969, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.921875, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2714844, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.62353516, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5507812, "text": " believes" }, { @@ -152,32 +152,32 @@ }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7402344, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21899414, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.76708984, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { @@ -187,47 +187,47 @@ }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6796875, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8222656, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.3544922, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24609375, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.02960205, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17358398, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3925781, "text": "\n" }, { @@ -237,7 +237,7 @@ }, { "id": 37, - "logprob": -4.59375, + "logprob": -4.5898438, "text": ":" }, { @@ -247,67 +247,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99365234, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2304688, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.107788086, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04257202, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024871826, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056915283, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.0071105957, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.008453369, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.0006928444, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074920654, "text": ":" } ], @@ -315,37 +315,37 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.828125, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.3178711, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.23925781, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.61279297, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.4177246, "special": false, "text": "Daniel" }, @@ -356,26 +356,26 @@ "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5283203, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007965088, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -390,22 +390,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -415,197 +415,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -625,67 +625,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -693,67 +693,67 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -768,22 +768,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -793,197 +793,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -1003,67 +1003,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -1071,67 +1071,67 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -1146,22 +1146,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -1171,197 +1171,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -1381,67 +1381,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -1449,66 +1449,66 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" } ] diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 44915ff5e..4fc4c3896 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -37,7 +37,7 @@ class FlashRW(FlashCausalLM): ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 + dtype = torch.float16 else: raise NotImplementedError("RW is only available on GPU") @@ -124,7 +124,7 @@ class FlashRWSharded(FlashRW): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 + dtype = torch.float16 else: raise NotImplementedError("FlashRW is only available on GPU") diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index dd389027e..2b1e49590 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -16,7 +16,7 @@ class RW(CausalLM): ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 + dtype = torch.float16 else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 030c8289a..134ac7cd3 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -23,7 +23,11 @@ def weight_hub_files( """Get the weights filenames on the hub""" api = HfApi() info = api.model_info(model_id, revision=revision) - filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] + filenames = [ + s.rfilename + for s in info.siblings + if s.rfilename.endswith(extension) and len(s.rfilename.split("/")) == 1 + ] if not filenames: raise EntryNotFoundError( @@ -130,24 +134,31 @@ def download_weights( ) -> List[Path]: """Download the safetensors files from the hub""" - def download_file(filename): + def download_file(filename, tries=5): local_file = try_to_load_from_cache(model_id, revision, filename) if local_file is not None: logger.info(f"File {filename} already present in cache.") return Path(local_file) - logger.info(f"Download file: {filename}") - start_time = time.time() - local_file = hf_hub_download( - filename=filename, - repo_id=model_id, - revision=revision, - local_files_only=False, - ) - logger.info( - f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." - ) - return Path(local_file) + for i in range(tries): + try: + logger.info(f"Download file: {filename}") + start_time = time.time() + local_file = hf_hub_download( + filename=filename, + repo_id=model_id, + revision=revision, + local_files_only=False, + ) + logger.info( + f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." + ) + return Path(local_file) + except Exception as e: + if i + 1 == tries: + raise e + logger.error(e) + logger.info(f"Retry {i + 1}/{tries - 1}") # We do this instead of using tqdm because we want to parse the logs with the launcher start_time = time.time()