Simple Encryptor HTB Challenge
A walkthrough of HackTheBox's SimpleEncryptor reverse engineering challenge. In this writeup I reverse engineer a 64-bit ELF binary using Ghidra, identify a custom encryption scheme based on XOR and bit rotation seeded by the system time, and exploit the fact that the seed is stored inside the encrypted file itself to recover the plaintext flag.
# Reconnaissance
The unzipped folder contains 2 files. "encrypt" and "flag.enc". I started by running the standard commands to learn more about the files.
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ cat flag.enc
Z5�b�>�����u���9�K�!�C#qe�'K%
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ file flag.enc
flag.enc: data
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ file encrypt
encrypt: ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=0bddc0a794eca6f6e2e9dac0b6190b62f07c4c75, for GNU/Linux 3.2.0, not stripped
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ strings encrypt
/lib64/ld-linux-x86-64.so.2
libc.so.6
srand
fopen
ftell
time
__stack_chk_fail
fseek
fclose
malloc
fwrite
fread
__cxa_finalize
__libc_start_main
GLIBC_2.4
GLIBC_2.2.5
_ITM_deregisterTMCloneTable
__gmon_start__
_ITM_registerTMCloneTable
u+UH
dH3<%(
[]A\A]A^A_
flag
flag.enc
:*3$"
GCC: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
crtstuff.c
deregister_tm_clones
__do_global_dtors_aux
completed.8061
__do_global_dtors_aux_fini_array_entry
frame_dummy
__frame_dummy_init_array_entry
encrypt.c
__FRAME_END__
__init_array_end
_DYNAMIC
__init_array_start
__GNU_EH_FRAME_HDR
_GLOBAL_OFFSET_TABLE_
__libc_csu_fini
_ITM_deregisterTMCloneTable
fread@@GLIBC_2.2.5
_edata
fclose@@GLIBC_2.2.5
__stack_chk_fail@@GLIBC_2.4
__libc_start_main@@GLIBC_2.2.5
srand@@GLIBC_2.2.5
__data_start
ftell@@GLIBC_2.2.5
__gmon_start__
__dso_handle
_IO_stdin_used
time@@GLIBC_2.2.5
__libc_csu_init
malloc@@GLIBC_2.2.5
fseek@@GLIBC_2.2.5
__bss_start
main
fopen@@GLIBC_2.2.5
fwrite@@GLIBC_2.2.5
__TMC_END__
_ITM_registerTMCloneTable
__cxa_finalize@@GLIBC_2.2.5
.symtab
.strtab
.shstrtab
.interp
.note.gnu.property
.note.gnu.build-id
.note.ABI-tag
.gnu.hash
.dynsym
.dynstr
.gnu.version
.gnu.version_r
.rela.dyn
.rela.plt
.init
.plt.got
.plt.sec
.text
.fini
.rodata
.eh_frame_hdr
.eh_frame
.init_array
.fini_array
.dynamic
.data
.bss
.comment
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ chmod +x encrypt
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ ./encrypt
Nothing special there, although I realized that there is the word flag and flag.enc, so I changed the file name to flag and it outputted another file called flag.enc with different content.
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ mv flag.enc flag
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ cat flag.enc
x�iM^
@LN!�xڋS���P)����8Sr�/U
;�%
Long story short this is unintended and is not part of the solution. I guess the owner of the challenge had the actual flag in a file called "flag", they then ran the executable and outputted the file "flag.enc". In hindsight this is obvious but oh well.
# Static Analysis
The tool used here is Ghidra.
I took the main function to a proper code editor for further analysis. This is the function;
undefined8 main(void)
{
int iVar1;
time_t tVar2;
long in_FS_OFFSET;
uint local_40;
uint local_3c;
long local_38;
FILE *local_30;
size_t local_28;
void *local_20;
FILE *local_18;
long local_10;
local_10 = *(long *)(in_FS_OFFSET + 0x28);
local_30 = fopen("flag","rb");
fseek(local_30,0,2);
local_28 = ftell(local_30);
fseek(local_30,0,0);
local_20 = malloc(local_28);
fread(local_20,local_28,1,local_30);
fclose(local_30);
tVar2 = time((time_t *)0x0);
local_40 = (uint)tVar2;
srand(local_40);
for (local_38 = 0; local_38 < (long)local_28; local_38 = local_38 + 1) {
iVar1 = rand();
*(byte *)((long)local_20 + local_38) = *(byte *)((long)local_20 + local_38) ^ (byte)iVar1;
local_3c = rand();
local_3c = local_3c & 7;
*(byte *)((long)local_20 + local_38) =
*(byte *)((long)local_20 + local_38) << (sbyte)local_3c |
*(byte *)((long)local_20 + local_38) >> 8 - (sbyte)local_3c;
}
local_18 = fopen("flag.enc","wb");
fwrite(&local_40,1,4,local_18);
fwrite(local_20,1,local_28,local_18);
fclose(local_18);
if (local_10 != *(long *)(in_FS_OFFSET + 0x28)) {
__stack_chk_fail();
}
return 0;
}
I will take the function and try to make sense of it. I will add comments where necessary and change the names of variables. This is the new version;
undefined8 main(void)
{
int iVar1;
time_t sysTime;
long in_FS_OFFSET;
uint time;
uint local_3c;
long i;
FILE *flagFile;
size_t flagLength;
void *flagContent;
FILE *local_18;
long local_10;
// Stack canary: reads a secret value from the thread/process control block.
// Checked at the end to detect stack buffer overflows.
local_10 = *(long *)(in_FS_OFFSET + 0x28);
// Open the plaintext flag file in read-binary mode
flagFile = fopen("flag","rb");
// Seek to the end of the file (offset 0 from end, SEEK_END=2)
fseek(flagFile,0,2);
// ftell returns current position = number of bytes from start = file size
flagLength = ftell(flagFile);
// Seek back to the beginning (offset 0 from start, SEEK_SET=0)
fseek(flagFile,0,0);
// Allocate heap memory equal to the file size
flagContent = malloc(flagLength);
// Read the entire file into the allocated buffer
fread(flagContent,flagLength,1,flagFile);
fclose(flagFile);
// Get current Unix timestamp
sysTime = time((time_t *)0x0);
time = (uint)sysTime;
// Seed the RNG with current time — means encryption is reproducible
// if you know the timestamp
srand(time);
// Loop over every byte of the flag
for (i = 0; i < (long)flagLength; i = i + 1) {
// Get a random value
iVar1 = rand();
// XOR the current byte with the low byte of the random value
*(byte *)((long)flagContent + i) = *(byte *)((long)flagContent + i) ^ (byte)iVar1;
// Get another random value and mask to range 0–7 (3 bits)
// This will be the rotation amount
local_3c = rand();
local_3c = local_3c & 7;
// Rotate the byte LEFT by local_3c bits
// Left shift OR right shift by (8 - local_3c) = standard 8-bit left rotation
*(byte *)((long)flagContent + i) =
*(byte *)((long)flagContent + i) << (sbyte)local_3c |
*(byte *)((long)flagContent + i) >> 8 - (sbyte)local_3c;
}
// Open output file flag.enc in write-binary mode
local_18 = fopen("flag.enc","wb");
// Write the timestamp (4 bytes) as the first 4 bytes of the file
// This is the decryption key — the seed needed to reproduce the RNG sequence
fwrite(&time,1,4,local_18);
// Write the encrypted flag content after the timestamp
fwrite(flagContent,1,flagLength,local_18);
fclose(local_18);
// Verify stack canary hasn't changed — if it has, someone smashed the stack
if (local_10 != *(long *)(in_FS_OFFSET + 0x28)) {
__stack_chk_fail();
}
return 0;
}
The code basically opens the file with the flag and figures out its length. It then allocates memory in the heap to store that contents of that file. It gets the system time (Epoch) and uses that value as the seed. It loops through the flag and does 2 things;
- XOR the byte with the low byte of the rand value
- Left bit rotation
Then it outputs the encrypted flag with the seed in the beginning.
# The Vulnerability
The vulnerability is that the seed is outputted in the .enc file provided. The encryption script depend entierly on the random value, kind of like a password. When computers generate random values (RNG), that outputted value is not truelly random since at the end of the day is done through mathematical equations. That random value is generated through a seed, if that seed is exposed that "random" value is regenerated. The seed is basically the decryption key.
If I did not have access to that value, I would consider brute-forcing the seed, I would know when the seed is correct if the first 3 characters of the decoded string is "HTB{" since that is how every flag on HTB is formatted.
# Solution
I have to create a script the does exactly the opposite of that script but with some changes.
My script would do the same thing but instead of generating a random value with the seed as the current time I would just take the time from the output of the earlier file. This is the script that I have crafted;
#include <stdio.h>
#include <stdlib.h>
int main(void)
{
int rand_1;
unsigned int time;
unsigned int rand_2;
long i;
FILE *flagFile;
size_t flagLength;
void *flagContent;
FILE *outputFile;
flagFile = fopen("flag.enc", "rb");
fseek(flagFile, 0, 2);
flagLength = ftell(flagFile) - 4;
flagContent = malloc(flagLength);
fseek(flagFile, 0, 0);
fread(&time, 1, 4, flagFile);
fread(flagContent, flagLength, 1, flagFile);
fclose(flagFile);
srand(time);
for (i = 0; i < (long)flagLength; i = i + 1) {
rand_1 = rand();
rand_2 = rand();
rand_2 = rand_2 & 7;
*(unsigned char *)((long)flagContent + i) =
*(unsigned char *)((long)flagContent + i) >> rand_2 |
*(unsigned char *)((long)flagContent + i) << (8 - rand_2);
*(unsigned char *)((long)flagContent + i) =
*(unsigned char *)((long)flagContent + i) ^ (unsigned char)rand_1;
}
outputFile = fopen("flag.dec", "wb");
fwrite(flagContent, 1, flagLength, outputFile);
fclose(outputFile);
free(flagContent);
return 0;
}
It follows the same structure as the encryption script, with three key differences: for example there is no new random value being created and used as the seed is the same as the old one.
I also print only the decrypted flag without the seed in the beginning.
Additionally, the steps are reversed, so in the original file the XORing is done first but in the new version I have it so that the the shift is done first.
Flipping the order of the operations done in the original encryption is necessary, think of it like a math equation, you would need to reverse the order of operation in order to get the same old value.
Finally I would run the executable.
someone@someone:~/HTB/ReverseEngineering/simple/rev_simpleencryptor$ gcc dec.c -o dec
./dec
cat flag.dec
HTB{vRy_s1MplE_F1LE3nCryp0r}
