The challenge gives an executable `vm` and a `bin` file that is a program that run in that vm. The usage is `./vm bin`. To understand the binary I began by analyzing the vm file in ghidra. The function to start the VM is as follows:
```Cmachine * vm_create(void *bin,void *bin_size)
{  machine *m;  void *mem_ptr;   m = (machine *)malloc(168);  m->pc_offset = 0;  m->should_halt = false;  m->set_to_zero = 0;  memset(m->initialized,0,128);  mem_ptr = calloc(0x10000,1);  m->instr_mem_0x10000 = mem_ptr;                    /* First 3 bytes do not go to mem, it`s a header "UwU"*/  memcpy(m->instr_mem_0x10000,(void *)((long)bin + 3),(long)bin_size - 3);  mem_ptr = calloc(0x200,4);  m->0x200_mem = mem_ptr;  return m;}```
Every single operation in this function was an offset from the same variable, so I created an `machine` struct to help to keep track of these ofsets as shown in the code above. Over the full course of the challenge, this is what I think the offsets mean:
|offset |type  | name|:---: | :---: | :---: ||0|int4|program_counter||1|bool|should_halt|8|byte[0x80]|ram_mem144|byte[0x10000]|instr_mem152|byte[0x200]|stack_mem160|int4|stack_pointer
Each vm operation is executed through this function:```Cvoid vm_step(machine *m)
{  byte opcode;   opcode = *(byte *)((ulong)(uint)m->pc_offset + (long)m->instr_mem_0x10000);  if (25 < cur_instr_addr) {    puts("dead");    exit(0);  }  (*(code *)original_ops[(int)(uint)opcode])(m);  return;}```This piece of code shows that `original_ops` is a function pointer array that describes all of the vm's instructions. These adresses still had their symbols, so the instruction names were: "vm_add", "vm_addi", "vm_sub", "vm_subi", "vm_mul", "vm_muli", "vm_div", "vm_cmp", "vm_jmp", "vm_inv", "vm_push", "vm_pop", "vm_mov", "vm_nop", "vm_exit", "vm_print", "vm_putc", "vm_je", "vm_jne", "vm_jle", "vm_jge", "vm_xor", "vm_store", "vm_load" and "vm_input".
These functions read like assembly, so it's easy to get an idea of what they do. To understand how they work however, it is necessary to do more reverse engineering. I did a lazy reverse engineering by only analysing the functions as they appear in the `bin` file. This python script creates a very rough disassembly:
```pythonfrom sys import argv
inst_list =  ["vm_add", "vm_addi", "vm_sub", "vm_subi", "vm_mul", "vm_muli", "vm_div", "vm_cmp", "vm_jmp", "vm_inv", "vm_push", "vm_pop", "vm_mov", "vm_nop", "vm_exit", "vm_print", "vm_putc", "vm_je", "vm_jne", "vm_jle", "vm_jge", "vm_xor", "vm_store", "vm_load", "vm_input"]
if len(argv) < 2:    print("missing filename")    exit
with open(argv[1], "rb") as fp:    file = fp.read()instr = file[3:]
pc = 0while pc < len(instr):    if instr[pc] == 0: # add        dest = hex(instr[pc+1])        arg1 = hex(instr[pc+2])        arg2 = hex(instr[pc+3])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{dest}] <= [{arg1}]+[{arg2}]")        pc += 6    elif instr[pc] == 1: # addi        dest = hex(instr[pc+1])        orig = hex(instr[pc+2])        imm = instr[pc+3]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{dest}] <= [{orig}]+{imm}")        pc += 6    elif instr[pc] == 5: # muli        dest = hex(instr[pc+1])        orig = hex(instr[pc+2])        imm = instr[pc+3]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{dest}] <= [{orig}]*{imm}")        pc += 6    elif instr[pc] == 9: # syscall        syscall  = hex(instr[pc+1])        num_param = instr[pc+2]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} syscall {syscall} with {num_param} parameters")        pc += 6    elif instr[pc] == 10: # push        val = hex(instr[pc+1])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} {val}")        pc += 6    elif instr[pc] == 11: # pop        val = hex(instr[pc+1])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{val}]")        pc += 6    elif instr[pc] == 12: # mov        pos = hex(instr[pc+1])        val = instr[pc+2]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{pos}] <= {val}")        pc += 6    elif instr[pc] == 14: # exit        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} (halt_flag=true)")        pc += 6    elif instr[pc] == 16: # putc        c = chr(instr[pc+1])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} {c}")        pc += 6    elif instr[pc] == 17: # je        arg1 = hex(instr[pc+1])        arg2 = hex(instr[pc+2])        arg3 = instr[pc+3]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} ([{arg1}] == [{arg2}])? goto {hex(6*arg3)} : goto {hex(pc+6)}")        pc += 6    elif instr[pc] == 19: # jle        arg1 = hex(instr[pc+1])        arg2 = hex(instr[pc+2])        arg3 = instr[pc+3]        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} ([{arg1}] <= [{arg2}])? goto {hex(6*arg3)} : goto {hex(pc+6)}")        pc += 6    elif instr[pc] == 21: # xor        dest = hex(instr[pc+1])        arg1 = hex(instr[pc+2])        arg2 = hex(instr[pc+3])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{dest}] <= [{arg1}]^[{arg2}]")        pc += 6    elif instr[pc] == 22: # store        pos = hex(instr[pc+1])        val = hex(instr[pc+2])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} code[{pos}] <= [{val}]")        pc += 6    elif instr[pc] == 23: # load        arg1 = hex(instr[pc+1])        arg2 = hex(instr[pc+2])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{arg1}] <= code[{arg2}]")        pc += 6    elif instr[pc] == 24: # input        pos = hex(instr[pc+1])        print(f"{hex(pc)}:\t{inst_list[instr[pc]]} [{pos}]")        pc += 6
    elif instr[pc] < 25:        print(f"unknown instruction at {hex(pc)}: {instr[pc]}!")        exit()    else:        print(f"{hex(pc)}: data - {chr(instr[pc])}")        pc += 1```
This is the disassembly of the `bin` file:
```0x0:	vm_putc [0x6:	vm_putc M0xc:	vm_putc a0x12:	vm_putc i0x18:	vm_putc n0x1e:	vm_putc  0x24:	vm_putc V0x2a:	vm_putc e0x30:	vm_putc s0x36:	vm_putc s0x3c:	vm_putc e0x42:	vm_putc l0x48:	vm_putc  0x4e:	vm_putc T0x54:	vm_putc e0x5a:	vm_putc r0x60:	vm_putc m0x66:	vm_putc i0x6c:	vm_putc n0x72:	vm_putc a0x78:	vm_putc l0x7e:	vm_putc ]0x84:	vm_putc '\n'0x8a:	vm_putc <0x90:	vm_putc  0x96:	vm_putc E0x9c:	vm_putc n0xa2:	vm_putc t0xa8:	vm_putc e0xae:	vm_putc r0xb4:	vm_putc  0xba:	vm_putc k0xc0:	vm_putc e0xc6:	vm_putc y0xcc:	vm_putc c0xd2:	vm_putc o0xd8:	vm_putc d0xde:	vm_putc e0xe4:	vm_putc  0xea:	vm_putc '\n'0xf0:	vm_putc >0xf6:	vm_putc  0xfc:	vm_mov [0x1e] <= 1600x102:	vm_mov [0x1c] <= 00x108:	vm_mov [0x1d] <= 170x10e:	vm_input [0x19]0x114:	vm_store code[0x1e] <= [0x19]0x11a:	vm_addi [0x1e] <= [0x1e]+10x120:	vm_addi [0x1c] <= [0x1c]+10x126:	vm_jle ([0x1c] <= [0x1d])? goto 0x10e : goto 0x12c0x12c:	vm_mov [0x1e] <= 40x132:	vm_mov [0x1f] <= 1600x138:	vm_mov [0x1c] <= 00x13e:	vm_mov [0x1d] <= 100x144:	vm_mov [0x1b] <= 1690x14a:	vm_mov [0x17] <= 00x150:	vm_load [0x19] <= code[0x1e]0x156:	vm_load [0x18] <= code[0x1f]0x15c:	vm_xor [0x19] <= [0x19]^[0x1b]0x162:	vm_je ([0x19] == [0x18])? goto 0x1d4 : goto 0x1680x168:	vm_putc U0x16e:	vm_putc n0x174:	vm_putc k0x17a:	vm_putc n0x180:	vm_putc o0x186:	vm_putc w0x18c:	vm_putc n0x192:	vm_putc  0x198:	vm_putc k0x19e:	vm_putc e0x1a4:	vm_putc y0x1aa:	vm_putc c0x1b0:	vm_putc o0x1b6:	vm_putc d0x1bc:	vm_putc e0x1c2:	vm_putc !0x1c8:	vm_putc '\n'0x1ce:	vm_exit (halt_flag=true)0x1d4:	vm_addi [0x1e] <= [0x1e]+10x1da:	vm_addi [0x1f] <= [0x1f]+10x1e0:	vm_addi [0x1c] <= [0x1c]+10x1e6:	vm_jle ([0x1c] <= [0x1d])? goto 0x150 : goto 0x1ec0x1ec:	vm_mov [0xf] <= 00x1f2:	vm_push 0xf0x1f8:	vm_push 0xf0x1fe:	vm_push 0xf0x204:	vm_inv syscall 0x65 with 3 parameters0x20a:	vm_mov [0x10] <= 00x210:	vm_je ([0x1f] == [0x10])? goto 0x288 : goto 0x2160x216:	vm_putc T0x21c:	vm_putc e0x222:	vm_putc r0x228:	vm_putc m0x22e:	vm_putc i0x234:	vm_putc n0x23a:	vm_putc a0x240:	vm_putc l0x246:	vm_putc  0x24c:	vm_putc b0x252:	vm_putc l0x258:	vm_putc o0x25e:	vm_putc c0x264:	vm_putc k0x26a:	vm_putc e0x270:	vm_putc d0x276:	vm_putc !0x27c:	vm_putc '\n'0x282:	vm_exit (halt_flag=true)0x288:	vm_mov [0x1e] <= 1190x28e:	vm_muli [0x1e] <= [0x1e]*60x294:	vm_mov [0x1c] <= 00x29a:	vm_mov [0x1d] <= 2200x2a0:	vm_mov [0x1b] <= 690x2a6:	vm_load [0x19] <= code[0x1e]0x2ac:	vm_xor [0x19] <= [0x19]^[0x1b]0x2b2:	vm_store code[0x1e] <= [0x19]0x2b8:	vm_addi [0x1e] <= [0x1e]+10x2be:	vm_addi [0x1c] <= [0x1c]+10x2c4:	vm_jle ([0x1c] <= [0x1d])? goto 0x2a6 : goto 0x2ca```
This code starts by getting the user pointer and then comparing to a string decoded with a xor key, being the comparison instruction the vm_je at 0x1e6. The easiest way I found to get the key was to put a breakpoint at vm_je with `bp vm_je` and defining a hook to change what I wrote with the right char and to print it on the screen.
```define hook-stopset *(int*)($rdi+0x68)=*(int*)($rdi+0x6c)p/c *(char*)($rdi+0x6c)end```
Then, running the vm with any input just running continue in gdb would give the key: `c0d3_r3d_5h`!
After that, the program checked if it was in a terminal with a syscall and exited if it was not. That is a problem for running gdb, but can easily be bypassed by changing the vm_je parameters with an hex editor, since giving the same adress twice guarantees equality.
The worst part is what comes next :(
The program decodes with a xor key some section of data and jumps to it. This means the binary has self modifying code and unpacks the part of itself which actualy tests for the flag.
To analyse this part of code I used a breakpoint to stop the vm in a struction after the unpacking was done and dumped the whole memory with `dump binary memory unpacked_bin *(char *)($rdi+144) *(char *)($rdi+144)+0x10000`. (To do this it necessary to do the gdb bypass described above).
Then I added the "UwU" header in the file and ran it through the disassembler again. The new piece code is as follows (this one needed more deep analysis so I had to do some formatting and commenting):
```; print enter secret phrase
; 0x2ca offset
0x90:	vm_mov [0x1e] <= 48 ; a0x96:	vm_mov [0x1c] <= 0  ; i0x9c:	vm_mov [0x1d] <= 36 ; max_iread_loop:    0xa2:	vm_input [0x19]    0xa8:	vm_store code[0x1e] <= [0x19]    0xae:	vm_addi [0x1e] <= [0x1e]+1    0xb4:	vm_addi [0x1c] <= [0x1c]+1  ; a++, i++0xba:	vm_jle ([0x1c] <= [0x1d])? goto read_loop : goto 0xc0
; for i in range(36):; 	read(0x19+i)
0xc0:	vm_mov [0x1c] <= 0  ; j0xc6:	vm_mov [0x1d] <= 35 ; max_jouter_loop:
    0xcc:	vm_mov [0x1e] <= 48     ; a    0xd2:	vm_mov [0x1f] <= 148    ; b    0xd8:	vm_mov [0x1a] <= 0      ; i    0xde:	vm_mov [0x1b] <= 35     ; max_i    inner_loop:        0xe4:	vm_load [0x14] <= code[0x1e]        0xea:	vm_load [0x15] <= code[0x1f]        0xf0:	vm_push 0x14        0xf6:	vm_pop [0x13]        0xfc:	vm_mov [0x12] <= 48        0x102:	vm_add [0x12] <= [0x12]+[0x15]        0x108:	vm_load [0x11] <= code[0x12]        ; adds 48 to b and reads memory        0x10e:	vm_store code[0x1e] <= [0x11] ; secret stored in b+48        0x114:	vm_store code[0x12] <= [0x13] ; user input        ; stores result on user input and stores user_input backup at b+48        0x11a:	vm_addi [0x1a] <= [0x1a]+1        0x120:	vm_addi [0x1e] <= [0x1e]+1        0x126:	vm_addi [0x1f] <= [0x1f]+1 	; i++ , a++, b++    0x12c:	vm_jle ([0x1a] <= [0x1b])? goto inner_loop : goto 0x132
    0x132:	vm_mov [0x1e] <= 48     ; a    0x138:	vm_mov [0x1f] <= 248    ; b_base    0x13e:	vm_mov [0x1a] <= 0      ; i    0x144:	vm_mov [0x1b] <= 35     ; max_i    second_loop:        0x14a:	vm_load [0x14] <= code[0x1e]        0x150:	vm_push 0x1f        0x156:	vm_pop [0xf]        0x15c:	vm_add [0xf] <= [0xf]+[0x1c]        0x162:	vm_load [0x10] <= code[0xf]     ; read b_base+j        0x168:	vm_xor [0x14] <= [0x14]^[0x10]        0x16e:	vm_store code[0x1e] <= [0x14]        0x174:	vm_addi [0x1a] <= [0x1a]+1        0x17a:	vm_addi [0x1e] <= [0x1e]+1      ; i++, a++    0x180:	vm_jle ([0x1a] <= [0x1b])? goto second_loop : goto 0x186
    0x186:	vm_addi [0x1c] <= [0x1c]+1  ; j++
0x18c:	vm_jle ([0x1c] <= [0x1d])? goto outer_loop : goto 0x192
0x192:	vm_mov [0x1e] <= 48     ; a0x198:	vm_mov [0x1f] <= 92     ; b0x19e:	vm_mov [0x1a] <= 0      ; i0x1a4:	vm_mov [0x1b] <= 35     ; max_iaccess_loop:    0x1aa:	vm_load [0xf] <= code[0x1e]    0x1b0:	vm_load [0x10] <= code[0x1f]    0x1b6:	vm_je ([0xf] == [0x10])? goto next : goto 0x1bc
    ; print Wrong    0x1e6:	vm_exit (halt_flag=true)
    next:    0x1ec:	vm_addi [0x1a] <= [0x1a]+1    0x1f2:	vm_addi [0x1e] <= [0x1e]+1    0x1f8:	vm_addi [0x1f] <= [0x1f]+1  ; i++, a++, b++0x1fe:	vm_jle ([0x1a] <= [0x1b])? goto access_loop : goto 0x204
; print acess granted0x2be:	vm_exit (halt_flag=true)```
This time there's a more complex verification that does a lot of xoring shuffling in the user input before the actual comparison, so just breaking at the comparison instruction and printing it's value does not work. I could try to understand what this code does and to do it in reverse, but I'm too lazy to do that and used a bruteforce aproach.
Before the bruteforce I did some changes to the `unpacked_bin` file. First I put a jump in the beginning to go right to the second user input read loop, and put a `vm_putc @` after the comparison at 0x1b6 to be able to track if I got a more partially correct input. This modified `unpacked_bin` is available here with after a gzip and base64 encoding:
```H4sICM9EGGQAA3NlY19yZWFkX21vZADt1MtLVGEYx/EzeSlHc86Z+80zY0aLKNAGpxYRthiCoFZpi5rEZiYtp9EsGYQg6B8Ioo1SFLRqE9EfELSKCndFCy+LNi6irQRtOu97yHnyDJEpbfp+Fj+P73nPe310uD5s+Xz3DMM8YzjMMZ1XddZ0ZnWO6KzovCnSbamKnudE+4zO654xx8RXRZ1+ncfFOAXR/5ZnTLfPpGif01nSOaWzLN5mxSwnGi1d9pOAk2n93GM5GU+o56itfvhs2+dkOu1kKN1zWPVvNZ3MyK/UoF2pZypjqiWWsFXGM05GEomUYViJ+FlnxmGxo0nx7K627jnzP99db2N33Sp+rjyTkes/5qwwoN76PdlRaVHrN9WzZaqb2fI9umu+LFqmxJrlXsrN19xlqyNos+32xtkut6mzLWycqnueTW/nxl/Uj6zhknhbEV+5Pad1Tog+Y54RmlRXv9XYy37jt9V1v9Gzr/Ft5oHKpG5PqfZYWFdXRFWXP6zaO0P6bVD1N4LBiPPWCupZVEM0qF77kknfr1URSqYeNWb51nQWf0aPryvECASc1cVM9UskHDbV+OFNIztjPhc7mm+MfzG4afyA3oWpJrAC5lvnxM6Ls50SdzfevFq8OzKHVLuzhlcbt1kStyP/a2XFyPI2vdXi1uohT824lTAr+rsp/zrGxVdlsS/5l+7ZVwGAlwEAwP+k1F/Ojc7kyqODEyc1/87P8W5+cWF9aWFx/eHr5ZXF+dU3Oz8FtiCUCPh3Gz3dUbPL15mJh/d0pJO9rfuCbSkr274rEttr97Vsd47oi1Mvfd9Xv9wtXiiat2u9o2uFg/m+D58OfHxqfV279H7p8crn7c5RKdZP54ZGqrNHi4O12pV8NT9dqV8bmCsOTN+p1nJH8vmB4nbnAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPDv+H8AqZHq5AQAAQA=```
The bruteforce script is as follows:
```pythonimport subprocessimport string
# dictionary of already discovered charsknown_chars = {}# number of @ printed (equals to chars that are right)num_ats = 0# number of chars in flagN_CHARS = 36
i = 0while num_ats < N_CHARS: # stops if all chars are correct
    # no reason to keep trying a known char    if i in known_chars.keys():        i = (i+1)%N_CHARS        continue
    # creates a new guess with known_chars and a random one    for c in string.printable:        cur_guess = []        for j in range(N_CHARS):            if j in known_chars.keys():                cur_guess.append(known_chars[j])            else:                cur_guess.append("A")        cur_guess[i] = c        guess = "".join(cur_guess)
        # calls function with current guess and counts '@'s        p = subprocess.run(["./vm", "sec_read_mod"], input=f"{guess}".encode(),capture_output=True)        if(p.stdout.count(b"@") > num_ats):            print(guess)            known_chars[i] = c            num_ats += 1            break
    # the tries may loop around the string because order is shuffled    i = (i+1)%N_CHARS```
After around a minute the script yields the flag: `HTB{5w1rl_4r0und_7h3_4l13n_l4ngu4g3}`