<!DOCTYPE html><html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" style="font-size:16px;"><head></head><head><meta charset="utf-8"/><!--[if !mso]><!--><meta http-equiv="X-UA-Compatible" content="IE=edge"/><!--<![endif]--><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="x-apple-disable-message-reformatting"/><meta name="format-detection" content="telephone=no,address=no,email=no,date=no,url=no"/><meta name="color-scheme" content="light"/><meta name="supported-color-schemes" content="light"/><title>Transformers without Normalization</title><!--[if mso]><xml><o:OfficeDocumentSettings><o:AllowPNG/><o:PixelsPerInch>96</o:PixelsPerInch></o:OfficeDocumentSettings></xml><![endif]--><style>
:root { color-scheme: light; supported-color-schemes: light; }
body { margin: 0; padding: 0; min-width: 100%!important; -ms-text-size-adjust: 100% !important; -webkit-transform: scale(1) !important; -webkit-text-size-adjust: 100% !important; -webkit-font-smoothing: antialiased !important; }
.body { word-wrap: normal; word-spacing:normal; }
table.mso { width: 100%; border-collapse: collapse; padding: 0; table-layout: fixed; }
img { border: 0; outline: none; }
table { mso-table-lspace: 0px; mso-table-rspace: 0px; }
td, a, span { mso-line-height-rule: exactly; }
#root [x-apple-data-detectors=true],
a[x-apple-data-detectors=true],
#MessageViewBody a { color: inherit !important; text-decoration: inherit !important; font-size: inherit !important; font-family: inherit !important; font-weight: inherit !important; line-height: inherit !important; }
span.MsoHyperlink { color: inherit !important; mso-style-priority: 99 !important; }
span.MsoHyperlinkFollowed { color: inherit !important; mso-style-priority: 99 !important; }
.a { background-color:#dedede; }
.b { background-color:#2a2a2a; }
.c { background-color:#ffffff; }
.d { background-color:#fff0c8; }
.d2 { background-color:#FFFFFF; }
.d3 { background-color:#FFFFFF; }
h1 a { text-decoration:none;color:#2A2A2A !important; }
h2 a { text-decoration:none;color:#2A2A2A !important; }
h3 a { text-decoration:none;color:#2A2A2A !important; }
h4 a { text-decoration:none;color:#2A2A2A !important; }
h5 a { text-decoration:none;color:#2A2A2A !important; }
h6 a { text-decoration:none;color:#2A2A2A !important; }
h1, h1 a, h2, h2 a, h3, h3 a, h4, h4 a, h5, h5 a, h6, h6 a, ul, li, ol, p, p a { margin: 0;padding: 0; }
h1 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:700;font-size:28px;color:#2A2A2A;line-height:42px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
h2 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:700;font-size:24px;color:#2A2A2A;line-height:36px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
h3 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:400;font-size:20px;color:#2A2A2A;line-height:30px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
h4 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:400;font-size:18px;color:#2A2A2A;line-height:27px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
h5 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:400;font-size:16px;color:#2A2A2A;line-height:24px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
h6 { font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif;font-weight:400;font-size:14px;color:#2A2A2A;line-height:21px;padding-bottom:4px;padding-top:16px;mso-margin-top-alt:16px;mso-margin-bottom-alt:4px }
p { font-family:'Georgia','Times New Roman',serif;font-weight:400;color:#2D2D2D;font-size:16px;line-height:24px;padding-bottom:8px;padding-top:8px;mso-margin-top-alt:8px;mso-margin-bottom-alt:8px; }
p a, .e a, ul a, li a, .h a, .h2 a, .h3 a { word-break:break-word;color:#2C81E5 !important;text-decoration:none;font-style:italic; }
p a span, .e a span, ul a span, li a span { color: inherit }
p .bold { font-weight:bold;color:#2D2D2D; }
p span[style*="font-size"] { line-height: 1.6; }
.f p { font-size:12px;line-height:15px;color:#2D2D2D;padding:0; }
.f p a { color:#2D2D2D !important; }
.g p { font-family:'Helvetica',Arial,sans-serif;font-size:14px;line-height:20px;font-weight:normal;margin:0; }
.g p a { text-decoration: underline; }
.i p { font-family:'Helvetica',Arial,sans-serif;line-height:23px;font-size:15px;color:#2D2D2D; }
.i p a { color:#2D2D2D !important; }
.i2 p { font-family:'Helvetica',Arial,sans-serif;line-height:23px;font-size:15px;color:#2D2D2D; }
.i2 p a { color:#2D2D2D !important; }
.i3 p { font-family:'Helvetica',Arial,sans-serif;line-height:43px;font-size:24px;color:#2D2D2D; }
.i3 p a { color:#2D2D2D !important; }
.h p a { color:#595959 !important; }
.h2 p a { color:#595959 !important; }
.h3 p a { color:#595959 !important; }
.f p a, .i p a, .i2 p a, .i3 p a, .h p a, .h2 p a, .h3 p a { text-decoration:underline; }
.j { border-top:3px solid #ffeb2d; }
.k p { padding-left:15px;padding-bottom:0px;padding-top:6px;mso-margin-top-alt:6px;mso-margin-bottom-alt:0px;mso-margin-left-alt:15px; }
.o { background-color:#FFFFFF;border:1px solid #F1F1F1;border-radius:5px; }
.o p { font-family:'Helvetica',Arial,sans-serif;padding:0px;margin:0px; }
.l p,
.l p a { font-size:14px;line-height:20px;font-weight: bold;color:#2D2D2D;padding-bottom:6px;mso-margin-bottom-alt:6px;text-decoration:none; }
.m p,
.m p a { font-size:13px;line-height:18px;font-weight:400;color:#2D2D2D;padding-bottom:6px;mso-margin-bottom-alt:6px;text-decoration:none; }
.n p,
.n p a { font-size:12px;line-height:17px;font-weight:400;color:#2D2D2D;padding-bottom:6px;mso-margin-bottom-alt:6px;text-decoration:none; }
.p { background-color:#FFFFFF;max-width:520px;border:1px solid #E1E8ED;border:1px solid rgba(80, 80, 80, 0.3);border-radius:5px; }
.q { font-size:16px;font-family:Helvetica,Roboto,Calibri,sans-serif !important;border:1px solid #e1e8ed;border:1px solid rgba(80, 80, 80, 0.3);border-radius:10px;background-color:#FFFFFF; }
.q p { font-size:16px;font-family:system-ui,Helvetica,Roboto,Calibri,sans-serif !important;color:#222222;padding:4px 0; }
.r { border:1px solid #E1E8ED !important;border-radius:5px; }
.s p { font-size: 14px; line-height: 17px; font-weight: 400; color: #697882; text-decoration: none; }
.t p { font-family:'Helvetica',Arial,sans-serif;font-size:12px;line-height:18px;font-weight:400;color:#000000;font-style:italic;padding:4px 0px 0px;}
.v { border-radius:10px;border:solid 0px #DFD150;background-color:#2C81E5;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;color:#FFFFFF; }
.v a { text-decoration:none;display:block;color:#FFFFFF; }
.w p { font-size:12px;line-height:15px;font-weight:400;color:#FFFFFF; }
.w p a { text-decoration: underline !important;color:#FFFFFF !important; }
ul { font-family:'Helvetica',Arial,sans-serif;margin:0px 0px 0px 25px !important;padding:0px !important;color:#2D2D2D;line-height:24px;list-style:disc;font-size:16px; }
ul > li { font-family:'Helvetica',Arial,sans-serif;margin:10px 0px 0px 0px !important;padding: 0px 0px 0px 0px !important; color: #2D2D2D; list-style:disc; }
ol { font-family:'Helvetica',Arial,sans-serif;margin: 0px 0px 0px 25px !important;padding:0px !important;color:#2D2D2D;line-height:24px;list-style:decimal;font-size:16px; }
ol > li { font-family:'Helvetica',Arial,sans-serif;margin:10px 0px 0px 0px !important;padding: 0px 0px 0px 0px !important; color: #2D2D2D; list-style:decimal; }
.e h3,
.e p,
.e span { padding-bottom:0px;padding-top:0px;mso-margin-top-alt:0px;mso-margin-bottom-alt:0px; }
.e span,
.e li { font-family:'Helvetica',Arial,sans-serif;font-size:16px;color:#2D2D2D;line-height:24px; }
.rec { font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji" !important; }
.rec__button:hover { background-color: #f9fafb !important; }
.copyright a {color: inherit !important; text-decoration: none !important; font-size: inherit !important; font-family: inherit !important; font-weight: inherit !important; line-height: inherit !important;}
.txt_social p { padding: 0; word-break: break-all; }
.table, .table-c, .table-h { border: 1px solid #C0C0C0; }
.table-c { padding:5px; background-color:#FFFFFF; }
.table-c p { color: #2D2D2D; font-family:'Helvetica',Arial,sans-serif !important;overflow-wrap: break-word; }
.table-h { padding:5px; background-color:#F1F1F1; }
.table-h p { color: #2A2A2A; font-family:'Trebuchet MS','Lucida Grande',Tahoma,sans-serif !important;overflow-wrap: break-word; }
@media only screen and (max-width:667px) {
.aa { width: 100% !important; }
.bb img { width: 100% !important; height: auto !important; max-width: none !important; }
.cc { padding: 0px 8px !important; }
.ee { padding-top:10px !important;padding-bottom:10px !important; }
.ff ul, .ff ol { margin: 0px 0px 0px 10px !important;padding: 0px !important; }
.ff li { margin:10px 0px 0px 10px !important; }
.r {height:140px !important;}
.s p { font-size:13px !important;line-height:15px !important; }
.mob-hide {display:none !important;}
.mob-stack {display:block !important;width:100% !important;}
.mob-w-full {width:100% !important;}
.mob-block {display:block !important;}
.embed-img {padding:0px 0px 12px 0px !important;}
.socialShare {padding-top:15px !important;}
.rec { padding-left:15px!important;padding-right:15px!important; }
.bodyWrapper { padding:7px 4px 7px 4px !important; }
.social-mobile {float:left !important;margin-top:10px !important;}
}
@media screen and (max-width: 480px) {
u + .a .gg { width: 100% !important; width: 100vw !important; }
.tok-heart { padding-top:75% !important; }
.tok-play { padding-top: 250px !important; }
}
@media screen and (max-width: 320px) {
.tok-heart { padding-top:65% !important; }
}
.u { border: 1px solid #CACACA !important; border-radius: 2px !important; background-color: #ffffff !important; padding: 0px 13px 0px 13px !important; font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif !important;font-size: 12px !important; color: #767676 !important; }
.u a { text-decoration: none; display: block !important; color: #767676 !important; margin: 0px !important; }
.u span, .u img { color: #767676 !important;margin:0px !important; max-height:32px !important;background-color:#ffffff !important; }
</style><!--[if mso]><style type="text/css">
sup { font-size: 100% !important;vertical-align: .5em !important;mso-text-raise: -1.5% !important;line-height: 0 !important; }
ul { margin-left:0px !important; margin-right:10px !important; margin-top:20px !important; margin-bottom:20px !important; }
ul li { margin-left: 0px !important; mso-special-format: decimal; }
ol { margin-left:0px !important; margin-right:10px !important; margin-top:20px !important; margin-bottom:20px !important; }
ol li { margin-left: 0px !important; mso-special-format: decimal; }
li.listItem { margin-left:15px !important; margin-top:0px !important; }
.paddingDesktop { padding: 10px 0 !important; }
.edm_outlooklist { margin-left: -20px !important; }
.embedImage { display:none !important; }
</style><![endif]--><style>
@font-face {
font-family: 'Open Sans';
font-style: normal;
font-weight: 700;
font-display: swap;
src: url('https://fonts.gstatic.com/s/opensans/v40/memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsg-1x4gaVIUwaEQbjA.woff2') format('woff2');
}
@font-face {
font-family: 'Open Sans';
font-style: italic;
font-weight: 700;
font-display: swap;
src: url('https://fonts.googleapis.com/css2?family=Open+Sans:ital,wght@1,700&display=swap') format('woff2');
}
</style></head><body class="a" style="margin:0px auto;padding:0px;word-wrap:normal;word-spacing:normal;background-color:#dedede;"><div role="article" aria-roledescription="email" aria-label="email_name" lang="en" style="font-size:1rem"><div style="display:none;max-height:0px;overflow:hidden;"> Plus more about RWKV-7 "Goose" with Expressive Dynamic State Evolution and Measuring AI Ability to Complete Long Tasks  ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ </div><table role="none" width="100%" border="0" cellspacing="0" align="center" cellpadding="0" class="gg"><tr><td align="center" valign="top"><table role="none" width="670" border="0" cellspacing="0" cellpadding="0" class="aa" style="width:670px;table-layout:fixed;"><tr><td class="bodyWrapper" align="center" valign="top" style="padding:7px 7px 7px 7px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" style="border:4px solid #2a2a2a;border-radius:0px;background-color:#ffffff;" class="c"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td class="f" align="right" valign="top" style="padding:20px 28px;"><p> March 25, 2025 | <a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.c6q0w4g5sodbtO4I1B_pxSdB5RCIH6yy1Fm1CYma3Eyc9WS40dQlm8xIjHAdExz2b5Uc7NZF6a7UjFgqU2M54-kefchOeqrcTzLYem9BpJoPvTaghXo79Q6Hw-wIhq9OI1Pmz70_mYIEyDy5X_N2D_aIBSImmvJHoPr7WebMl-0XtagFrOXoi3I4gy87BhL9pCA-odA5LRjTiDjnwl_jHwuIB_yaW1Bo-tGAkXH1L46m2fn8vNMHc_WFDVaxEJ0GnEhmE468rNArYILQ02po06UHBeA-efP1SkXQvHdQn_fHeMax7cK_xqtKl9kaYlBD28NLkqhdYROCQ_-6MIDxKfRGKBU8O7_ciRmfkqZSJJ44ZMAKDrwbTolhfOJv800dEXMgDq1kgN3UpJzGXrQW-abtgsfOKj6HAfhreGi3ns0WsGmIxX3WnMn6YoWqRpD7iVFaOKf7RQJl6mNQ1kW-LAs6Sf2jAvs0_KaF1amxAjvNpjF2sXyWYnrFF5RGAYmF7ZfCfyxpeocT6Mw9Wz9ngjaeThwrEFW4ji323UE5jTAdHI7yi5uiEuU_3JyFDsJWpKhKQKJA4W1COFa3fRCpZkgMJrook6K_Y3kxV2BaM9q8RYbGAFToG7Zn7fTHTCt_2fMJo0uIw8-TNU0_HkTOI1BvMhd8srVLwoJO5R8QsvI/4f3/FNFzSjpjTTGsHysY-7P5UA/h0/h001.U-eKEclAoBFalHsp9-8rcBHzvS2mSkcut2EXo-txWZs">Read Online</a></p></td></tr><tr><td class="dd" align="center" valign="top" style="padding:15px 28px 20px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top"><h1 style="text-align:left;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;font-weight:Bold;font-size:32px;color:#2A2A2A;padding:2px 0;line-height:38px;"> Transformers without Normalization </h1><p style="text-align:left;font-family:'Helvetica',Arial,sans-serif;font-weight:normal;font-size:20px;color:#3E3E3E;padding:5px 0;line-height:24px;"> Plus more about RWKV-7 "Goose" with Expressive Dynamic State Evolution and Measuring AI Ability to Complete Long Tasks </p></td></tr></table></td></tr><tr><td style="height:0px;width:0px;"><div style="height:1px;" data-open-tracking="true"> <img src="https://elink4f7.mail.bycloud.ai/ss/o/u001.3wmUuY8gEWd4_869a_eXcg/4f3/FNFzSjpjTTGsHysY-7P5UA/ho.gif" alt="" width="1" height="1" border="0" style="height:1px !important;width:1px !important;border-width:0 !important;margin-top:0 !important;margin-bottom:0 !important;margin-right:0 !important;margin-left:0 !important;padding-top:0 !important;padding-bottom:0 !important;padding-right:0 !important;padding-left:0 !important;"/> </div></td></tr><tr id="content-blocks"><td class="email-card-body" align="center" valign="top" style="padding-bottom:28px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td id="nov-18-th-nov-24-th-33-latest-ai-re" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h6 style="color:#2A2A2A;font-weight:normal;"><i>Mar 17th ~ Mar 23rd</i><br><i>#48 Latest AI Research Explained Simply</i></h6></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="#222222" style="background-color:#222222;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"></p></td></tr></table></td></tr></table></td></tr><tr><td id="industry-news-in-1-line" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;">🗞️ Industry News in 1 Line</h2></td></tr><tr><td style="padding-bottom:12px;padding-left:50px;padding-right:40px;padding-top:12px;" class="ee"><div style="margin-left:0px;" class="edm_outlooklist"><ol start="1" style="list-style-type:decimal;margin:0px 0px;padding:0px 0px 0px 0px;"><li class="listItem ultext"><p style="line-height:24px;padding:0px;text-align:left;word-break:break-word;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;">♥ 1.1k</span></span> <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.zNfxTwpJFmrsCuJJphGRkO8nGcnU28eUcMtPIqN_Fs1pEsCMu5O4Hvtbk7Zq83iwazDoV4nRGS4ccr6tSEAb0eYnU2cou82TGVs8kbx78HuhndHSH03aiZfPNcbp5Boe/4f3/FNFzSjpjTTGsHysY-7P5UA/h1/h001.0MKWuTrzz26YeyXQpGPxOsRSDXAMSp7jNRYT7MPdvC4" target="_blank" rel="noopener noreferrer nofollow"><span>Reve Image</span></a> released a new state-of-the-art image generator. This is their first model debut, with Reve Image performs very well on text rendering, prompt adherence, and aesthetics. Its “text rendering” is <i>extremely </i>good at generate text in images as you can see below. </p><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:420px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/a15dd0f6-fb6a-4efb-a44d-a1d12251513a/Gm1atv8WQAIFbcg.jpg?t=1742921461" alt="" height="auto" width="420" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:420px;"><p>generated image taken from fofr <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.1muhFWIqieRYpaJ-FbWSCdBtyepDZapGy9dezEVlREP7OjQjwIXPJ5Ml7AYBz6JNOIAfXY0481dURxRy4TWXMbhEwkI-9GE7-eoZjHGLxiEp9KQpAsHhN5JK58ZjU3724QPRV99K-l8iAp3hfOoAng/4f3/FNFzSjpjTTGsHysY-7P5UA/h2/h001.Iv5RFXsKs-u9hp0mjetTppX8Gy2WCoL3W1HY1tjrsyY" target="_blank" rel="noopener noreferrer nofollow"><span>https://x.com/fofrAI/status/1904278031448895904</span></a></p></td></tr></table><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:510px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/f11ee654-fc61-4e4d-a980-443050c07f6d/Screenshot_2025-03-25_124952.png?t=1742921402" alt="" height="auto" width="510" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:510px;"><p>Artificial Analysis Image Arena Leaderboard</p></td></tr></table></li><li class="listItem ultext"><p style="line-height:24px;padding:0px;text-align:left;word-break:break-word;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;">♥ 5.9k</span></span> <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.1muhFWIqieRYpaJ-FbWSCcRyRSdAB7_wsm52EZCEq6xeltHFfJtaVNp5L2qh3DPV4df1mF0tN91raKt03dYW3rBhOx05PXaK4hTQ8oNHwfuHUTGfoZx5VOK-r8mjsJBnLkonrGkh11Cb4bdIdknu1A/4f3/FNFzSjpjTTGsHysY-7P5UA/h3/h001.qhaup1St_YhkTBXC-0w4943Osc_tYNqVkcPeTlB0f1I" target="_blank" rel="noopener noreferrer nofollow"><span>DeepSeek released DeepSeek-V3-0324</span></a>, an updated non-reasoning model of DeepSeek-V3, and is now the state-of-the-art LLM for non-reasoning models excluding Claude 3.7 hybrid model. It is now the new best open source model, with its <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWoNV4Z0gzLAqdXCVgcyx3wsmEMg5nw7JnMGZ7SvhfwkV_3TS97LLZDSGuCL6fdbers5xpo7FbP5RyRbRRCcAoqntp3gr3pPFhu1KimWB3Ehkgk2iDp0ig5zhMNv0ypbwEw/4f3/FNFzSjpjTTGsHysY-7P5UA/h4/h001.HWWTB4wSdeqFyj2grCjrnO5jlcS1Jzms76KjCgYDDBI" target="_blank" rel="noopener noreferrer nofollow"><span>weights available on HuggingFace</span></a>. </p><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:480px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/ea060397-c099-4aea-aea7-edd33eadd6a7/Gm48k3XbkAEUYcN.jpg?t=1742921614" alt="" height="auto" width="480" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:480px;"><p>DeepSeek-V3-0324 benchmark</p></td></tr></table></li><li class="listItem ultext"><p style="line-height:24px;padding:0px;text-align:left;word-break:break-word;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;">♥ 2.1k</span></span> <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.fUNb4GdFo9D3F8WuLArtoUNDG9fZSIPOlPQYZvQFeD_Lt6xxy0m1Htl18D2uaaEbXArVRFC_uIBIGOT7_Gw4HaimiBLRTCb3w6qNrQQ35QQ/4f3/FNFzSjpjTTGsHysY-7P5UA/h5/h001.aUbaHba-mo9vbdUtqlb0oMI_9NE798nozRJJwhg8azk" target="_blank" rel="noopener noreferrer nofollow"><span>ARC Prize released ARC-AGI-2</span></a>, a second iteration of “AGI” benchmark that is even more challenging than the current ARC-AGI benchmark, with a grand prize of $700,000 that <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.fUNb4GdFo9D3F8WuLArtoUE9NY1VyXCvG1ua6HVArhJN6iv9P-R7Z7tdW48zxvpR0YCaPLdbQoGxoL_N-AJbtzWmzkiQiGzQ0PoG9W8iZFTvqkwl5AlLXn5xn0OCujXo/4f3/FNFzSjpjTTGsHysY-7P5UA/h6/h001.XpOoRr0ADwy5K3AXjypjFqzVQP9CPTcywDqFPha59y0" target="_blank" rel="noopener noreferrer nofollow"><span>anyone can participate in</span></a>. </p><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:420px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/23969af0-f476-4705-afef-508f20826d03/Gm1P6s4bMAAeM2r.png?t=1742921841" alt="" height="auto" width="420" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:420px;"><p>example from ARC-AGI-2</p></td></tr></table></li></ol></div></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="#222222" style="background-color:#222222;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"></p></td></tr></table></td></tr></table></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="transparent" style="background-color:transparent;border-color:#2C81E5;border-style:solid;border-width:5px;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;"><span style="">Support My Newsletter</span></h2></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style="color:rgb(34, 34, 34);font-family:Georgia, "Times New Roman", serif;font-size:16px;">As I aim to keep this newsletter free forever, your support means a lot. If you like reading The AI Timeline, consider forwarding it to another research enthusiast, It helps us keep this up for free!</span></p></td></tr><tr><td align="center" valign="top"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" style="font-size:0px;line-height:0px;padding:30px 0px;" class="dd"><table class="j" role="none" width="50%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td> </td></tr></table></td></tr><tr><td class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;">Share The AI Timeline</h2></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> You currently have <strong>0</strong> referrals. </p></td></tr><tr><td align="left" valign="top" style="padding: 20px 28px 20px; display:none;width:0px;max-height:0px;overflow:hidden;mso-hide:all;height:0;font-size:0;max-height:0;line-height:0;margin:0 auto;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 0;"><tr><td align="center" valign="top" style="width:300px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.c6q0w4g5sodbtO4I1B_pxWc4htTObwdorovK0nFHVH-4pUdVE0ELYH5DsNemk732SjNwhPNJ25r0O8B5vYifsGNUqyW5TiZkyMsF1yreu0byy2KW36J1wDdpoLuXg2TU1F1OW8OHoHaU4-ZmrZpPU4RN-crQCEimD190CSn9fPuxpIRojBJyu1VfV5KtQD3QMVdSg2JrjEj5-xm4r4E12Whf08itqPCb9Q5W0X4rt3ubYkqCmWnLeZpmb3_RZcbIk0UE5wZnFLCQJHLFs0qZ0OGpXp89o1HU4mWIBur5Or4tQGm5M_Y8m5PvTEfYfxLRyrcRv7GyVs5oLtFfiySZ2SqtZypLA-h50h61p0uPiA7iA_PiMqlVLtM-87XL33VZi05_O3UTpWE_0nAzFRJ4TW1ayz3_vn4Zlp9IERdbnnAd_1kPLD4lAQcR5PRXgtpC_KyQvFPdoN79BG86M2bhmSfah3KLHZOYE0d36DDJBDLSC4fj8cs0GNZTnVWrCL9OXkLCD7WdvfutehG0J5pcoTKP8RkYW8Z18FyPuQt3cRT1YUP-QVq6cbIVn7ZV0F4gw3t7wRBy6erMZnHhNqVLJxq-R-A-VKg8tay8bqc9CwrD4y3MH9wx3CKSXUnXwydk/4f3/FNFzSjpjTTGsHysY-7P5UA/h7/h001.Ns7PnubUT4yZUcPO9QvYZeOz0O_Oab19m_fIV-KAO5U" rel="noopener noreferrer nofollow" style="text-decoration:none;" target="_blank"><img src="" alt="" height="auto" width="300" style="display:block;width:100%;" border="0"/></a></td></tr></table></td></tr><tr><td align="left" valign="top" style="padding-bottom:14px;padding-left:28px;padding-right:28px;padding-top:14px;text-align:left;width:100%;word-break:break-word;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" align="left" style=""><tr><td align="center" valign="middle" height="42" style="height:42px;background-color:#2C81E5;border-radius:10px;border:solid 0px #DFD150;color:#FFFFFF;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.c6q0w4g5sodbtO4I1B_pxWc4htTObwdorovK0nFHVH-4pUdVE0ELYH5DsNemk732SjNwhPNJ25r0O8B5vYifsGNUqyW5TiZkyMsF1yreu0byy2KW36J1wDdpoLuXg2TU1F1OW8OHoHaU4-ZmrZpPU4RN-crQCEimD190CSn9fPuxpIRojBJyu1VfV5KtQD3QMVdSg2JrjEj5-xm4r4E12Whf08itqPCb9Q5W0X4rt3ubYkqCmWnLeZpmb3_RZcbIk0UE5wZnFLCQJHLFs0qZ0OGpXp89o1HU4mWIBur5Or4tQGm5M_Y8m5PvTEfYfxLRyrcRv7GyVs5oLtFfiySZ2SqtZypLA-h50h61p0uPiA7iA_PiMqlVLtM-87XL33VZi05_O3UTpWE_0nAzFRJ4TW1ayz3_vn4Zlp9IERdbnnAd_1kPLD4lAQcR5PRXgtpC_KyQvFPdoN79BG86M2bhmSfah3KLHZOYE0d36DDJBDLSC4fj8cs0GNZTnVWrCL9OXkLCD7WdvfutehG0J5pcoTKP8RkYW8Z18FyPuQt3cRT1YUP-QVq6cbIVn7ZV0F4gw3t7wRBy6erMZnHhNqVLJxq-R-A-VKg8tay8bqc9CwrD4y3MH9wx3CKSXUnXwydk/4f3/FNFzSjpjTTGsHysY-7P5UA/h8/h001.6weFdX9zOicqazLjqAF7htsdp8YSS0MiHkt6a1m-4RE" target="_blank" rel="noopener noreferrer nofollow" style="color:#FFFFFF;display:block;font-size:16px;padding:0px 14px;text-decoration:none;"> Click to Share </a></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> Or copy and paste this link to others: <a class="link" href="https://mail.bycloud.ai/subscribe?ref=6SqUHb8KiF&_bhlid=7fecfad9eb7fd8bcdb529e945e11346b5897acdc" target="_blank" rel="noopener noreferrer nofollow" clicktracking="off"><span>https://mail.bycloud.ai/subscribe?ref=6SqUHb8KiF</span></a></p></td></tr><tr><td align="center" valign="top" style="font-size:0px;line-height:0px;padding:30px 0px;" class="dd"><table class="j" role="none" width="50%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td> </td></tr></table></td></tr></table></td></tr><tr><td align="center" valign="top" style="padding-bottom:14px;padding-left:28px;padding-right:28px;padding-top:14px;text-align:center;width:100%;word-break:break-word;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" align="center" style=""><tr><td align="center" valign="middle" height="42" style="height:42px;background-color:#2C81E5;border-radius:10px;border:solid 0px #DFD150;color:#FFFFFF;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.zNfxTwpJFmrsCuJJphGRkKSrCVph9-fOYkcjx4VfJRwtQQsKrZC8pi-PiKai2fq4lAto9WepTJo69aQJ1T73b1BYaJHeCrLz1cWpFYfpKjdJ071BkzwRo9IrCS5YAIxy/4f3/FNFzSjpjTTGsHysY-7P5UA/h9/h001.khAo70j5i4TcsKxqSm-Bk1nex6lUs8ErptBxZLnzEQQ" target="_blank" rel="noopener noreferrer nofollow" style="color:#FFFFFF;display:block;font-size:16px;padding:0px 14px;text-decoration:none;"> Check Out My Patreon </a></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style=""><a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.tLfGW26lAwaS9gFg17HSoGymQ3NNPtd5dE5MV_8UgjIDFPVXngz8pvQBldSW42yhUe_Qiq6DgEPMEBuPL9yfRpXelTiuu2kS8pLFvsoem_XoZoy_n13sTKUhZIbl0VH6/4f3/FNFzSjpjTTGsHysY-7P5UA/h10/h001.7a0HO4jAks6CaCaWitID6bXfO_6rIE6HMDDkdFrjI4Q" target="_blank" rel="noopener noreferrer nofollow"><span>Advertise with The AI Timeline! </span></a></span></p></td></tr></table></td></tr></table></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="#222222" style="background-color:#222222;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"></p></td></tr></table></td></tr></table></td></tr><tr><td id="measuring-ai-ability-to-complete-lo" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;">Measuring AI Ability to Complete Long Tasks </h2></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style=""><i>Kwa et al. [Model Evaluation & Threat Research]</i></span></p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;"> ♥ 4.2k </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span><span style="background-color:#e0e0e0;"><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> LLM Agents </span></span></p></td></tr><tr><td id="why-is-ai-getting-better-at-complex" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Why is AI Getting Better At Complex Tasks?</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> AI models are getting better but we hardly know what are the real-world implications of this increase in AI benchmark performance. This paper introduced a novel metric, the 50%-task-completion time horizon, to address the challenge of understanding the real-world implications of AI benchmark performance. The metric measures the <span style="font-weight:700;"><b>time humans typically take to complete tasks</b></span> that AI models can complete with a <span style="font-weight:700;"><b>50% success rate</b></span>. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/4fb5d8b2-95d7-40a9-a9a3-26d75e5b0606/image.png?t=1742916220" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td id="comparing-ai-against-humans-on-comp" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Comparing AI Against Humans on Complex Tasks</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> In this study, the researchers designed a task suite to measure AI agent performance on realistic tasks, comprising three distinct sets: a subset of HCAST with 97 diverse software tasks ranging from 1 minute to 30 hours, RE-Bench with 7 challenging machine learning research engineering tasks each taking about 8 hours, and Software Atomic Actions (SWAA) with 66 single-step tasks representing short segments of software engineering work, ranging from 1 second to 30 seconds. They automatically scored each task and grouped them into task families to maintain diversity and account for correlated performance. The researchers designed the tasks to be <span style="font-weight:700;"><b>realistic and economically useful</b></span>, requiring skills that professionals in relevant domains would possess. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/aa4897cc-a83a-4acd-9f0f-12a995c00f2a/image.png?t=1742916436" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> To establish a baseline for AI performance, the researchers measured the performance of over <span style="font-weight:700;"><b>800 human</b></span> "baseliners" across the tasks, totaling 2,529 hours. These baseliners were skilled professionals in software engineering, machine learning, and cybersecurity, with an average of about 5 years of relevant experience. For HCAST tasks, the researchers used existing baselines, while for RE-Bench, they utilized baselines from its original paper. They baselined SWAA tasks using a custom webapp for precise timing. The researchers calculated task durations and success thresholds from the human baseline data, providing a grounded comparison for AI agent performance. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/29672415-3808-4953-9791-fdc2e4adb001/image.png?t=1742916240" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>Methodology for measuring AI agent time horizon</p></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The researchers evaluated the AI models using consistent agent scaffolds across the task suites, with minimal task-specific prompting. They performed approximately 8 runs per agent/task pair and observed a strong upward trend in performance over time, with recent models completing about 50% of all tasks. The researchers found a significant negative correlation between the time it takes a human baseliner to complete a task and the average success rate of AI models on that task, which they well-fitted with an exponential model. This comprehensive methodology provides valuable insights into the capabilities and limitations of current AI systems on realistic tasks. </p></td></tr><tr><td align="center" valign="top" style="padding:14px 32px 14px 32px;" class="dd"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.VomAAYwkCjux8i_FMc4kJYvPZNO6ipCJDxBiGdbL6r--A4GCJoXwXW7_0Kcozqddo_y948WRrpupmDheQmwfT5qeJK9da1Io0rVSw7coHan0RwXZ1SuOBKl9ww17ObPCNtqZw7HUPpwHWXx6Kuyggw/4f3/FNFzSjpjTTGsHysY-7P5UA/h11/h001.gq7XM4ViS-MW6TQKEBwms6oaBjl3dGhxfif-8tbbckw" style="text-decoration:none;" target="_blank"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" class="o" style="padding:12px 12px;"><table role="none" border="0" cellspacing="0" cellpadding="0" align="right" width="100%"><tr><!--[if mso]><td width="0"><table cellpadding="0" cellspacing="0" border="0" role="presentation" style="display:none;"><tr><![endif]--><td class="embed-img" align="center" valign="top" style="width:100%;min-height:100px;vertical-align:middle;padding:0px 0px 12px 0px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.VomAAYwkCjux8i_FMc4kJYvPZNO6ipCJDxBiGdbL6r--A4GCJoXwXW7_0KcozqddTgK60Pj6S9h3Ds-FiYUpbcYlClsYa7njKiOHQ9co-z303iTb6XLMc8SY0dEM0qHI87VEJiY74nFaeVdfG3w8yg/4f3/FNFzSjpjTTGsHysY-7P5UA/h12/h001.zdl_PlQyQpo7BxnF8o_hubqaxDdmIlJnnM5dTzuJp8Y" style="text-decoration:none;" target="_blank"><img src="https://opengraph.githubassets.com/edab23f4aaf9968eb1460fe54c97e251ea022a0a2ddce0de702bb664f1951366/METR/eval-analysis-public" width="100%" style="display:block;"/></a></td><!--[if mso]></tr></table></td><![endif]--></tr><tr><td align="center" valign="top" class="cc"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="left" valign="top" class="l"><p>GitHub - METR/eval-analysis-public: Public repository containing METR's DVC pipeline for eval data analysis</p></td></tr><tr><td align="left" valign="top" class="m"><p>Public repository containing METR's DVC pipeline for eval data analysis - METR/eval-analysis-public</p></td></tr><tr><td align="left" valign="bottom" class="n" style="vertical-align:bottom;padding-top:12px;"><p style="word-break:break-word;">github.com/METR/eval-analysis-public</p></td></tr></table></td></tr></table></td></tr></table></a></td></tr><tr><td id="can-ai-models-replace-humans-in-com" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Can AI Models Replace Humans in Complex Tasks?</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The benchmark results show that newer AI models significantly outperform older ones, particularly in tasks involving machine learning training, reverse engineering, and cybersecurity challenges. We found that current models excel in tasks requiring situational awareness and the ability to adapt to mistakes, which demonstrates improved tool use capabilities and better logical reasoning and code generation. However, AI agents still struggle in "messier" environments where feedback loops are unclear or where they need to proactively seek information. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/3c393a82-fa20-446f-98a2-b1b228fcd242/image.png?t=1742916410" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> When comparing the failures of older models like GPT-4 with newer ones like o1, we noticed that over a third of GPT-4's failures were due to <span style="font-weight:700;"><b>repeating unsuccessful actions</b></span>, while o1 showed a marked improvement in adapting to mistakes. Interestingly, o1's failures often resulted from <span style="font-weight:700;"><b>prematurely abandoning tasks</b></span>, possibly due to tackling more challenging tasks. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/3bb603c3-e13f-4535-a38e-1e1b869ae93d/image.png?t=1742916478" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> While there are still limitations, especially in less structured environments, the overall trend suggests that AI systems are becoming increasingly capable and reliable. If these trends continue, AI could soon automate many tasks currently performed by humans, and revolutionize fields like software engineering and research. </p></td></tr><tr><td align="center" valign="top" style="padding-bottom:14px;padding-left:28px;padding-right:28px;padding-top:14px;text-align:center;width:100%;word-break:break-word;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" align="center" style=""><tr><td align="center" valign="middle" height="42" style="height:42px;background-color:#2C81E5;border-radius:10px;border:solid 0px #DFD150;color:#FFFFFF;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.fUNb4GdFo9D3F8WuLArtoV5sElgytBlvJRzI9WtI92Zxh8tDN5cH2PN6UyN6orXEQV_pYc9zKtH8eLWQlRBDOWXg-ZIo3RcOX0cEJCFhUqDowrnUTrNMlWlwWZxMF3EH/4f3/FNFzSjpjTTGsHysY-7P5UA/h13/h001.gyqDhsZg7cZmD7rTFqeLag_b8NKS_dLNeZEfnOCAKuc" target="_blank" rel="noopener noreferrer nofollow" style="color:#FFFFFF;display:block;font-size:16px;padding:0px 14px;text-decoration:none;"> Read Full Paper </a></td></tr></table></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="#222222" style="background-color:#222222;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"></p></td></tr></table></td></tr></table></td></tr><tr><td id="rwkv-7-goose-with-expressive-dynami" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;">RWKV-7 "Goose" with Expressive Dynamic State Evolution </h2></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style=""><i>Peng et al. [RWKVProject (under Linux Foundation AI & Data), EleutherAI, Tsinghua University, Recursal AI, Dalle Molle Institute for Artificial Intelligence USI-SUPSI, Guangdong Laboratory of Artificial Intelligence and Digital Economy (SZ), George Mason University, New York University, Tano Labs, Shenzhen University, University of Oslo, Beijing Normal University, Denigma]</i></span></p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;"> ♥ 683 </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span><span style="background-color:#e0e0e0;"><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> Linear Attention </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span></p></td></tr><tr><td align="center" valign="top" style="padding:14px 32px 14px 32px;" class="dd"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWrDOXQovq5HcCNu9kAcjY2BsHORuK0ZmX82-y_D-Le9dqzxsitHxsneWUu-v4XJUkw3h1rRte_-HCFt48okU9_AEvL8kz0kpCJCdj7jQOYkpDp7J3gEhELPmenIgempJhQ/4f3/FNFzSjpjTTGsHysY-7P5UA/h14/h001.mM2F_9o9RZVwdYi_bpEfXgLcJOO2ku76pyH0fYjdlQs" style="text-decoration:none;" target="_blank"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" class="o" style="padding:12px 12px;"><table role="none" border="0" cellspacing="0" cellpadding="0" align="right" width="100%"><tr><!--[if mso]><td width="0"><table cellpadding="0" cellspacing="0" border="0" role="presentation" style="display:none;"><tr><![endif]--><td class="embed-img" align="center" valign="top" style="width:100%;min-height:100px;vertical-align:middle;padding:0px 0px 12px 0px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWrDOXQovq5HcCNu9kAcjY2BsHORuK0ZmX82-y_D-Le9dqzxsitHxsneWUu-v4XJUk38_ozJBwt3NjLISQ3aR0JwrdynNIJLE4c5bYLazVHgn9eDviqmFGjxnoBJhaKJTWg/4f3/FNFzSjpjTTGsHysY-7P5UA/h15/h001.5k9mHeQtVvWyHnqLaRj8TYLtMJ1NDnYcMsTbMqs4hqI" style="text-decoration:none;" target="_blank"><img src="https://cdn-thumbnails.huggingface.co/social-thumbnails/models/RWKV/RWKV7-Goose-World3-2.9B-HF.png" width="100%" style="display:block;"/></a></td><!--[if mso]></tr></table></td><![endif]--></tr><tr><td align="center" valign="top" class="cc"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="left" valign="top" class="l"><p>RWKV/RWKV7-Goose-World3-2.9B-HF · Hugging Face</p></td></tr><tr><td align="left" valign="top" class="m"><p>We’re on a journey to advance and democratize artificial intelligence through open source and open science.</p></td></tr><tr><td align="left" valign="bottom" class="n" style="vertical-align:bottom;padding-top:12px;"><p style="word-break:break-word;">huggingface.co/RWKV/RWKV7-Goose-World3-2.9B-HF</p></td></tr></table></td></tr></table></td></tr></table></a></td></tr><tr><td id="introduction-to-rwkv-7-goose" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Introduction to RWKV-7 "Goose"</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The RWKV-7 "Goose" paper introduces a new <span style="font-weight:700;"><b>sequence modeling architecture</b></span> that significantly improves performance in language tasks, especially in multilingual settings, while using fewer training tokens than other models of similar size. This new model not only matches the best English language performance but also sets a new standard at the 3 billion parameter scale for multilingual tasks. </p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> RWKV-7 has the ability to maintain constant memory usage and inference time per token, this directly addresses the growing computational costs seen in traditional Transformer models as sequence lengths increase. The paper also presents a massive new 3.1 trillion token <span style="font-weight:700;"><b>multilingual dataset</b></span>, RWKV World v3, which was used to train these models. The dataset and the training as well as the inference code is openly available under the Apache 2.0 License. </p></td></tr><tr><td align="center" valign="top" style="padding:14px 32px 14px 32px;" class="dd"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWoLE8R9BmZi5_D0_25nOCBpBv_iUpauOKyunZpkqCJibzp11HWJ7XHfw0qMIHCp89kY5jEdSpgIDXXqtYTeP6Eq4aGJgj8mNi7AhopSOL1wBYL94iQd1h-3djzlBcXS8vg/4f3/FNFzSjpjTTGsHysY-7P5UA/h16/h001.QMFoAUSTaBcmLLFVG_JKu-3Z6naMJ1uhFJoj7nyNFv8" style="text-decoration:none;" target="_blank"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" class="o" style="padding:12px 12px;"><table role="none" border="0" cellspacing="0" cellpadding="0" align="right" width="100%"><tr><!--[if mso]><td width="0"><table cellpadding="0" cellspacing="0" border="0" role="presentation" style="display:none;"><tr><![endif]--><td class="embed-img" align="center" valign="top" style="width:100%;min-height:100px;vertical-align:middle;padding:0px 0px 12px 0px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWoLE8R9BmZi5_D0_25nOCBpBv_iUpauOKyunZpkqCJibzp11HWJ7XHfw0qMIHCp89sCxU_ENqzKWVuEib35u0zZSoPSVmzmyNPuX1O-x8wj_-BCvshSNfEfMT4QRu_Pc7g/4f3/FNFzSjpjTTGsHysY-7P5UA/h17/h001.kjo9oG0cpWJTk7CnsBjxZXpO7rMUBsubQ3xf3o1DoaY" style="text-decoration:none;" target="_blank"><img src="https://cdn-thumbnails.huggingface.co/social-thumbnails/datasets/RWKV/RWKV-World-Listing.png" width="100%" style="display:block;"/></a></td><!--[if mso]></tr></table></td><![endif]--></tr><tr><td align="center" valign="top" class="cc"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="left" valign="top" class="l"><p>RWKV/RWKV-World-Listing · Datasets at Hugging Face</p></td></tr><tr><td align="left" valign="top" class="m"><p>We’re on a journey to advance and democratize artificial intelligence through open source and open science.</p></td></tr><tr><td align="left" valign="bottom" class="n" style="vertical-align:bottom;padding-top:12px;"><p style="word-break:break-word;">huggingface.co/datasets/RWKV/RWKV-World-Listing</p></td></tr></table></td></tr></table></td></tr></table></a></td></tr><tr><td id="how-does-rwkv-7-goose-work" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">How does RWKV-7 "Goose" Work?</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The RWKV-7 architecture is a new approach to sequence modeling that improves upon existing methods. It uses a generalized delta rule, which is a way of updating the model's state based on the input data. This rule is more flexible and powerful than previous versions, allowing the model to capture more complex patterns in the data. The architecture also includes a number of other innovations, such as a vector-valued decay mechanism, which helps to control the amount of information that is retained in the model's state over time. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/b5dd47df-dbb9-4c15-9a7d-bb6d9067d03d/RWKV-paper.png?t=1742916707" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The model's state is updated using a combination of two mechanisms: a <span style="font-weight:700;"><b>decay</b></span> mechanism, which reduces the importance of older information, and a <span style="font-weight:700;"><b>replacement</b></span> mechanism, which adds new information to the state. The decay mechanism is controlled by a vector-valued parameter, which allows the model to selectively forget certain types of information. The replacement mechanism is also controlled by a vector-valued parameter, which allows the model to selectively add new information to the state. </p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The RWKV-7 architecture is designed to be highly parallelizable, which makes it efficient to train and use. It also has a number of other advantages, such as the ability to recognize regular languages and perform state tracking, which are important tasks in natural language processing. </p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The researchers also introduced a new dataset called RWKV World v3, which is a large multilingual dataset that is designed to provide excellent English, code, and multilingual capabilities. They trained four RWKV-7 models on this dataset, ranging from <span style="font-weight:700;"><b>0.19 billion to 2.9 billion parameters</b></span>, and achieved state-of-the-art results on a number of benchmarks. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/dcdd3195-7684-4bd4-81af-e9138a23956a/RWKV-formula.png?t=1742916748" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td id="results-and-evaluation" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Results and Evaluation</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The VisualRWKV-7 model has powerful generation capabilities, it surpasses the previous VisualRWKV-6 model on several benchmarks, including VQAv2 and GQA, with only a <span style="font-weight:700;"><b>quarter of the parameters</b></span>. The model's performance on out-of-domain benchmarks also shows strong generalization ability. The RWKV-7 architecture achieves state-of-the-art performance for its size across a wide range of benchmarks, which makes it a compelling alternative to traditional Transformer-based architectures. </p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> However, the model still faces limitations, such as numerical precision issues, lack of instruction tuning and alignment, prompt sensitivity, and limited compute resources. The future models trained on the RWKV-7 architecture have the potential to rival highly optimized models if they are trained on bigger dataset with more parameters. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/e9997494-c5f5-4fa0-80ed-41219573abca/RWKV-eval2.png?t=1742916874" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>RWKV [loss vs token position] for 10000 ctx4k+ documents in Pile.</p></td></tr></table></td></tr><tr><td align="center" valign="top" style="padding-bottom:14px;padding-left:28px;padding-right:28px;padding-top:14px;text-align:center;width:100%;word-break:break-word;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" align="center" style=""><tr><td align="center" valign="middle" height="42" style="height:42px;background-color:#2C81E5;border-radius:10px;border:solid 0px #DFD150;color:#FFFFFF;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.DUiN96-Eq7pUHzwEhy5j28yjf9KIXZdsXoh1WlHvvKmww30wH_LBlBlqs_AB4n-FRQcLLOuP-ioW3qSv0-iZx4Yamyk2hLHwSTLV7rAfbPRgRM-QpFHG5unASLm0Xq8X/4f3/FNFzSjpjTTGsHysY-7P5UA/h18/h001.Vl_1u3mOX27uSrxpnK6URd42WNp8_NUI8NpJQXMH3b0" target="_blank" rel="noopener noreferrer nofollow" style="color:#FFFFFF;display:block;font-size:16px;padding:0px 14px;text-decoration:none;"> Read Full Paper </a></td></tr></table></td></tr><tr><td><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" style=""><tr><td bgcolor="#222222" style="background-color:#222222;padding:0.0px 0.0px 0.0px 0.0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0"><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"></p></td></tr></table></td></tr></table></td></tr><tr><td id="transformers-without-normalization" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:Bold;padding:0px 28px;text-align:left;"><h2 style="color:#2A2A2A;font-weight:Bold;">Transformers without Normalization</h2></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style=""><i>Zhu et al. [FAIR, Meta, New York University, MIT, Princeton University]</i></span></p></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"><span style="background-color:#e0e0e0;"><span style="color:rgb(255, 58, 58);font-size:0.6rem;"> ♥ 4.1k </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span><span style="background-color:#e0e0e0;"><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> LLM Architecture </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span><span style="background-color:#e0e0e0;"><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> bycloud’s pick </span></span><span style="color:rgb(44, 129, 229);font-size:0.6rem;"> </span></p></td></tr><tr><td id="can-transformers-work-without-norma" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Can Transformers Work without Normalization</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> Normalization layers are currently considered essential for training deep neural networks, especially Transformers. This paper challenges this assumption by introducing Dynamic Tanh (DyT), a simple element-wise operation that replaces normalization layers. DyT mimics the input-output mapping of layer normalization by scaling activations and squashing extreme values using a learnable parameter and the tanh function. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/2c91b420-270f-4b8d-af3b-5e50439b8aa7/image.png?t=1742916957" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>Left: original Transformer block. Right: block with our proposed Dynamic Tanh (DyT) layer.</p></td></tr></table></td></tr><tr><td id="inner-workings-of-transformers-with" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Inner-Workings of Transformers without Normalization</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The researchers tested the behavior of Layer Normalization (LN) in trained Vision Transformer (ViT), wav2vec 2.0, and Diffusion Transformer (DiT) models. By analyzing the input-output mappings of LN layers, they observe a predominantly linear relationship in early layers, transitioning to a tanh-like, S-shaped curve in deeper layers. This non-linearity was somewhat unexpected given the linear nature of mean and standard deviation calculations within LN. This non-linearity arises from the per-token normalization: each token's activations are linearly transformed, but the varying scales and offsets across tokens collectively produce the tanh-like curve. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/6596b376-f2c8-4403-a2d5-9b0ca31e26ef/image.png?t=1742917004" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>Output vs. input of selected layer normalization (LN) layers in Vision Transformer (ViT), wav2vec2.0(a Transformer model for speech), and Diffusion Transformer (DiT).</p></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> This S-shaped curve effectively squashes extreme activation values, bringing them closer to the mean while largely preserving the linear transformation for the majority of activations near zero. This squashing effect on outliers is known to be a key contributor to the effectiveness of normalization layers, potentially mimicking the saturation behavior observed in biological neurons. Further analysis reveals that different channels contribute distinct segments to the overall tanh curve, with channels exhibiting extreme values being squashed the most. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/2f02d107-759c-4b4b-a847-249212e08767/image.png?t=1742917129" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> Inspired by these observations, the paper proposes Dynamic Tanh (DyT) as a replacement for LN. DyT applies a scaled tanh function element-wise to the input tensor, DyT(x) = γ * tanh(αx) + β, where α is a learnable scaling parameter, and γ and β are learnable per-channel scaling and shifting parameters. By using DyT, the researchers can replicate the squashing effect of LN without computing activation statistics. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/730b577c-5ced-4282-8b9a-fcede61cf242/image.png?t=1742917150" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> </p></td></tr><tr><td id="testing-transformers-without-normal" class="dd" align="left" valign="top" style="color:#2A2A2A;font-weight:normal;padding:0px 28px;text-align:left;"><h3 style="color:#2A2A2A;font-weight:normal;"><span style="color:rgb(67, 67, 67);">Testing Transformers without Normalization</span></h3></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The researchers tested the new approach and found out that DyT <span style="font-weight:700;"><b>significantly speeds up AI model training</b></span> and inference, which is demonstrated by substantial improvements in LLaMA 7B benchmarks. During testing, the researchers found out that the <span style=""><i>tanh</i></span> function within DyT is suitable for training stability as it outperforms other squashing functions. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/1fa9b7da-3cdd-4526-9e4f-8fe2e592dd36/image.png?t=1742917222" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>Self-supervised learning accuracy on ImageNet-1K. DyT performs on par with LN across different pre-training methods and model sizes in self-supervised learning tasks.</p></td></tr></table></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/73399ca0-7031-45f4-9930-d52148d0938b/image.png?t=1742917367" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>Inference and training latency (BF16 precision) for LLaMA 7B with RMSNorm or DyT. DyT achieves a substantial reduction in both inference and training time.</p></td></tr></table></td></tr><tr><td class="dd" align="left" style="padding:0px 28px;text-align:left;word-break:break-word;"><p style="line-height:24px;"> The learnable scaling parameter α is essential for good performance as it dynamically adjusts to the input data characteristics similar to 1/std. α acts as a normalization mechanism, though differently than Layer Normalization. DyT offers a competitive alternative to other methods that eliminate normalization layers, which typically rely on specialized initialization or weight constraints. This makes DyT a promising choice for developers seeking both efficiency and performance in their AI models. </p></td></tr><tr><td align="center" valign="top" style="padding: 20px 28px 20px; " class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" style="margin:0 auto 0 auto;"><tr><td align="center" valign="top" style="width:600px;"><img src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/uploads/asset/file/22609f67-60bc-41d6-b59e-e549172a5512/image.png?t=1742917477" alt="" height="auto" width="600" style="display:block;width:100%;" border="0"/></td></tr><tr><td align="center" valign="top" class="t" style="width:600px;"><p>ImageNet-1K classification accuracy with different squashing functions.</p></td></tr></table></td></tr><tr><td align="center" valign="top" style="padding-bottom:14px;padding-left:28px;padding-right:28px;padding-top:14px;text-align:center;width:100%;word-break:break-word;" class="dd"><table role="none" border="0" cellspacing="0" cellpadding="0" align="center" style=""><tr><td align="center" valign="middle" height="42" style="height:42px;background-color:#2C81E5;border-radius:10px;border:solid 0px #DFD150;color:#FFFFFF;font-family:'Open Sans','Segoe UI','Apple SD Gothic Neo','Lucida Grande','Lucida Sans Unicode',sans-serif;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.fUNb4GdFo9D3F8WuLArtoV5sElgytBlvJRzI9WtI92Zzp5GlmmsQQmqU5NMqqZoOnS-VFqIIpJ0KbQ11krQI212tJIRSSEXFesW456dx4EMcw6xJ4-vVJShH7kfKqf3r/4f3/FNFzSjpjTTGsHysY-7P5UA/h19/h001.lfSDS-nert0osFoTfmMtPBT5vN1lJinUdvRmKWrGJG0" target="_blank" rel="noopener noreferrer nofollow" style="color:#FFFFFF;display:block;font-size:16px;padding:0px 14px;text-decoration:none;"> Read Full Paper </a></td></tr></table></td></tr><tr><td class="dd" style="padding: 20px;"><table width="100%" cellpadding="0" cellspacing="0" role="none" style="max-width:520px;margin:0 auto;"><tr><td class="q" style="padding:16px 16px 6px 16px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.tLfGW26lAwaS9gFg17HSoDDFT6eh5Nsg0xYVQj-h6I3o9m2k79_qw4izMYhmcI36F5ZISgWMgUrsUbkXMVVmFIx9lBRSxy9-yv3tC66e_v1MtLrzZye6Mc66QMmdnmGO9203WgqHjp1Q_zy5JEbEXggiESe_YKj_3qRDKsoHclE/4f3/FNFzSjpjTTGsHysY-7P5UA/h20/h001.sXB1whnI4ci_dxy0ufs3oSzusO1P_cHYDoJ9t2vAE8I" style="text-decoration:none !important;"><table width="100%" cellpadding="0" cellspacing="0" border="0" role="none"><tr><td width="100%" style="padding: 0 0 14px 0;text-decoration:none;width:100%;"><table width="100%" cellpadding="0" cellspacing="0" border="0" role="none"><tr><td width="36" style="width:36px;"><img src="https://pbs.twimg.com/profile_images/1698572487909400576/BvncwnrP_normal.jpg" alt="tw profile: The AI Timeline" style="display:block;width:36px;height:36px;border-radius:50%;border:0;"/></td><td width="400" style="padding:0 0 0 8px;text-decoration:none;"><span style="display:block;font-size:14px;color:#1c2022;font-weight:700;"> The AI Timeline </span><span style="display:block;color:#697882;font-size:14px;"> @TheAITimeline </span></td><td width="24" align="right" style="vertical-align:text-top;"><img width="24" height="24" loading="lazy" alt="tw" style="border:0;" src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/static_assets/x_logo.png"/></td></tr></table></td></tr><tr></tr><tr><td style="word-break:break-word;"><p>🚨 Last 2 week's top AI/ML research papers:</p><p>- Transformers without Normalization <br>- Block Diffusion <br>- Compute Optimal Scaling of Skills <br>- DAPO: An OS LLM RL System at Scale <br>- Teaching LLMs How to Learn with Contextual Fine-Tuning <br>- GR00T N1 <br>- Why the Brain Cannot Be a Digital</p></td></tr><tr><td style="padding:12px 0 0 0;"></td></tr><tr><td align="center" style="padding:8px 0 0 0;width:480px;"><img src="https://pbs.twimg.com/media/Gmq6-QAbcAAIM67.jpg" width="480" height="auto" style="display:block;border:1px solid #E1E8ED;border-radius:5px;width:100%;max-width:480px;height:auto;"/></td></tr><tr><td height="8" style="line-height:1px;font-size:1px;height:8px;"> </td></tr><tr><td align="left" valign="top" class="s"><p>8:16 PM • Mar 22, 2025</p></td></tr><tr><td height="10" style="line-height: 1px; font-size: 1px; height: 10px;"> </td></tr><tr><td height="1" bgcolor="#e1e8ed" style="line-height:0px;font-size:0px;height:1px;"></td></tr><tr><td height="10" style="line-height:1px;font-size:1px;height:10px;"> </td></tr><tr><td align="left" valign="top" class="s"><p><b style="color:#1C2022">1.22K</b> Likes <b style="color:#1C2022">165</b> Retweets </p></td></tr><tr><td align="left" valign="top" class="s"><div align="center" style="text-align:center;margin-top:4px;margin-bottom:4px;padding:8px;border:1px solid #ccd6dd;border-radius:9999px;color:#1B95E0"><b>5 Replies</b></div></td></tr></table></a></td></tr></table></td></tr><tr><td class="dd" align="center" valign="top" style="padding:20px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.amatuKKICSickUKplYJXmJVdP228RyZ6CYFWDNsbH14m3twqzoFUKaet0ix_DP6DQLCPP-pSgUl7VklAwFV3zfZJgfTho1quwNilyHKBIoWJDSJVyL6wzGYJ26_BKOn6/4f3/FNFzSjpjTTGsHysY-7P5UA/h21/h001.CEQfYdBr6OSKMq_UcAIyjBKqogcSV8p9Kk4PmO7GIRw" style="text-decoration:none;"><table align="center" width="100%" cellpadding="0" cellspacing="0" border="0" role="none" style="max-width:520px;margin:0 auto;"><tr><td class="p" width="100%" style="padding:2px;border:none;"><table width="100%" cellpadding="0" cellspacing="0" border="0" role="none"><tr><td align="center" valign="top" style="width:100%;"><div style="max-height:0;position:relative;opacity:0.999;width:100%;mso-hide:all;"><div style="display:inline-block;width:100%;padding-top:25%;"><img width="20%" height="auto" loading="lazy" alt="" style="border:0;" src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/static_assets/youtube_play_icon.png"/></div></div><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.amatuKKICSickUKplYJXmJVdP228RyZ6CYFWDNsbH14AOhcyrrlcslYXAUbP_3LiOMA48wHhH9wfUN8beymhMJNfBJSW3PvFxMHLW0vJsumxvmrzRHLblNu6bUuEFVCY/4f3/FNFzSjpjTTGsHysY-7P5UA/h22/h001.NH7-QXBcI5jN1Xxy1G5YxWiHCuu6_eUNZ7ge9DVJ5z4" style="text-decoration:none;"><img src="https://i.ytimg.com/vi/48GRiu-TMmg/maxresdefault.jpg" width="480" height="auto" loading="lazy" alt="YouTube video by bycloud" style="display:block;height:auto;border:0;outline:none;text-decoration:none;background-color:#000000;width:100%;"/></a></td></tr><tr><td><p style="font-size:12px;font-weight:500;font-style:italic;font-family:Helvetica, Calibri, sans-serif;color: #686a6d; padding-top:0 !important;padding-bottom:6px !important; padding-left:4px !important;"> What DeepSeek’s Open Source Week Means For AI [Full Breakdown] </p></td></tr></table></td></tr></table></a></td></tr></table></td></tr><tr><td class="b" align="center" valign="top" bgcolor="#2a2a2a" style="padding:0px;border-bottom-left-radius:0px;border-bottom-right-radius:0px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top" bgcolor="#73ddff" style="padding:12px"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td><span style="padding-left:1px;"></span></td><td align="center" valign="middle" width="75" style="width:75px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.1muhFWIqieRYpaJ-FbWSCQqcWoV4NNHHr5SkP9THApWuHAAlWLQxI3Q_IqFmt_DcyAxeC8jDApCnHmMSBGpBb5sgtimvBYgxRX-Rp7s0F3LjCHoSwdhr83OBqRFhJ1y_/4f3/FNFzSjpjTTGsHysY-7P5UA/h23/h001.8qjPrrSiJHnWR1yqPjH_vklfa5vWVnDB6u0ai0GvNEQ" style="text-decoration:none;"><img width="22" alt="tw" border="0" style="display:block;max-width:22px;" src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/static_assets/x_dark.png"/></a></td><td align="center" valign="middle" width="75" style="width:75px;"><a href="https://elink4f7.mail.bycloud.ai/ss/c/u001.amatuKKICSickUKplYJXmBoQnQ9VXnB2zTxBG4HeHBgjMqVxpoXRdj01cjwyoVlHgiebEOgBvwHtevoVpsSvpn3Q1di2ml6sb3cBM-X6IStQbj_zQSVGWJ8AAmPw2en2/4f3/FNFzSjpjTTGsHysY-7P5UA/h24/h001.cy8s1kpsB4Az5wS6_7lQUORe-DYjFkEtF36h2B8rJhs" style="text-decoration:none;"><img width="22" alt="yt" border="0" style="display:block;max-width:22px;" src="https://media.beehiiv.com/cdn-cgi/image/fit=scale-down,format=auto,onerror=redirect,quality=80/static_assets/youtube_dark.png"/></a></td><td><span style="padding-left:1px;"></span></td></tr></table></td></tr><tr><td height="10" style="line-height:1px;font-size:1px;height:10px;"> </td></tr><tr><td class="w" align="center" valign="top" style="padding:15px;"><table role="none" width="100%" border="0" cellspacing="0" cellpadding="0" align="center"><tr><td align="center" valign="top"><p style="font-family:'Verdana',Geneva,sans-serif;color:#FFFFFF!important;"> Update your email preferences or unsubscribe <a class="link" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.c6q0w4g5sodbtO4I1B_pxWc4htTObwdorovK0nFHVH-4pUdVE0ELYH5DsNemk732SjNwhPNJ25r0O8B5vYifsBhEpz-DJgyVFmavJPa0OyKRRnvw4o7XGyvIv7PRofnmyRhOUltBHuUu97b3gUc5IkFjJFcPOj_miiIocbdYmN7vZnq2yJLkEN5xoh5pvqDQGwPmS9sPDC0BTM7ydlP7dAEMqA-c3TkibZ9ticziqGPhqSs62ZjycGVjBLUh2mDQoUZBkJ2kQiSfPXJQuU5GPTUIt1f76c7nojpQY5rzla_Us6MKlGzV1PMxFc4uSgwwULBEJnoyOTLw71EbFVUv2jDinHDMQoALXCGPiyRNBZzu6Cg8V3nmVbgOT_dnt8cvgDxyuR1hz_KZ0MeSKulrOfckFr0TSRmbvax37TiKtAyMjetbwyMXF7PH-1azwv7bGh10xEoi2ODtMy1Gwfgugfo73TKuLoknIKJ7NQg7S1ZKCg5O9Zb2mycg0gq0fxNWdoLWdvyiSvFQNxN3maYKpSi5T6y3S3Ln2af_sSFHE9Stnlh9bh35jVj6nxN_60fxMk5dXoPpD4zML4OVet6B-TI4vw6Nk_L_PvsU-xkcXC3suH2wmUl4-p6AWit21ZHjvHr5ha86WNzYBWPveO69GNNW-jjeiXJmGYcgEVAJ5szTNa5V3qF7Kc4p5F9-LH0BmnQfXrGxk3VmlkCVSdlqQNc5-0ez_ucDCJEhcQV7kOV-9iCUT1QzbFAXb-8iiNJdQJU-z4DOPqmFYTJQuwIemldmrZtYXBr7BKwFPZZxzS5n_EXFe8ADjI3wZhn356HYPeF5Ry-_6xhE1g1Mrif2aA/4f3/FNFzSjpjTTGsHysY-7P5UA/h25/h001.dKnETITy6y8fqaTn3V1vBCYoEVtvTTZ5of7Rug2r0q4" style="text-decoration:underline;text-decoration-color:#FFFFFF!important;color:#FFFFFF!important;"> here</a></p><p class="copyright" style="font-family:'Verdana',Geneva,sans-serif;color:#FFFFFF!important;"> © 2025 bycloudai </p><p style="font-family:'Verdana',Geneva,sans-serif;color:#FFFFFF!important;"> 228 Park Ave S, #29976, New York, New York 10003, United States </p></td></tr><tr style="display: table-row !important;"><td align="center" valign="top" style="padding-top:20px;" style="display:table-cell !important;"><table role="none" border="0" cellspacing="0" cellpadding="0" align="center" style="display:table !important;"><tr style="display:table-row !important;"><td class="u" align="center" valign="middle" height="32" style="height:32px;display:table-cell !important; max-height: 32px !important;margin:0px !important; background-color: #ffffff !important;"><a style="line-height:32px !important;text-decoration:none;display:block !important;" href="https://elink4f7.mail.bycloud.ai/ss/c/u001.DUiN96-Eq7pUHzwEhy5j28olDWFpV5DDKfdk_OdOKOjTOiARuloqc2h5SfyicozDnAEHFtwitCdje-E8KqOaQkQs1BngL09bzMIvVVI3NGQRSN_EFZsm6yDH-YwSF_zOgTtFBt3DzEnqs9qpx0f4TAjGKkVyiOZRLCGLQ7RSGvf0BMdoZqccC95qYuIc_GKQ-TUYABohB9FmX5gpVYWfkChbMDfob7Fq3Y22meCszHsmBVO9u_KUa45iHZ6t2zp_/4f3/FNFzSjpjTTGsHysY-7P5UA/h26/h001.M_v1P1MRv44VgjzWRv3zgoAmGtp0wHj1BXjBRXWQdGQ"><img src="https://media.beehiiv.com/output-onlinepngtools.png" width="16" alt="beehiiv logo" style="display:inline-block !important;max-width:16px !important; vertical-align:-3px !important;width: 16px !important;" border="0"/><span style="padding-left:11px !important;display: inline-block !important;">Powered by beehiiv</span></a></td></tr></table></td></tr><tr><td align="left" valign="top" height="2" style="height:2px;"><a href='https://elink4f7.mail.bycloud.ai/ss/c/u001.CxDkkVpJsBdVoe83c_tBWsHIaP4XNp0WgUYqLvHcKk_3uqk_KIkz4ddLinhFbud6JuxLFdSUhYnR7b1NSsmbtzXNGNblnEEMKUtkCAjkn8Y/4f3/FNFzSjpjTTGsHysY-7P5UA/h27/h001.vG2BTHQUaCDRumNod8dx74VbSAfO5NkxHd45Kwv0jQA' style="color: #2a2a2a !important; cursor: default; font-size: 1px; text-decoration: none;"> Terms of Service </a></td></tr></table></td></tr></table></td></tr></table></td></tr></table></td></tr></table></td></tr></table></div></body></html>