Merge pull request #36 from hashtopolis/bug/windows-path

Adding test framework, fix for window paths and fix for preprocessors

Author: zyronix

.devcontainer/Dockerfile

@@ -43,7 +43,9 @@ ENV DEBIAN_FRONTEND=dialog
 WORKDIR /app
 USER vscode
 COPY requirements.txt /app/src/
+COPY requirements-tests.txt /app/src/
 RUN /usr/bin/pip3 install -r src/requirements.txt
+RUN /usr/bin/pip3 install -r src/requirements-tests.txt
 
 # Check for and run optional user-supplied command to enable (advanced) customizations of the dev container
 RUN if [ -n "${DEV_CONTAINER_USER_CMD_POST}" ]; then echo "${DEV_CONTAINER_USER_CMD_POST}" | sh ; fi

.devcontainer/windows/Dockerfile

@@ -0,0 +1,20 @@
+FROM mcr.microsoft.com/windows-cssc/python3.7.2server:ltsc2022
+# Nano image doesn't work because some API are not available
+
+# TODO: Support for USER_CMD_PRE and POST?
+# TODO: Create a vscode user?
+# TODO: OpenCL/Nvidia?
+
+WORKDIR C:/App/
+
+# Installing python requirements
+COPY requirements.txt C:/App/
+COPY requirements-tests.txt C:/App/
+RUN pip3 install -r requirements.txt -r requirements-tests.txt
+
+# Fix for host.docker.internal not working
+COPY .devcontainer/windows/entrypoint.ps1 C:/
+COPY .devcontainer/windows/fix-hosts.ps1 C:/
+
+# Setting entrypoint
+ENTRYPOINT "C:\entrypoint.ps1"

.devcontainer/windows/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3"
+services:
+  hashtopolis-agent-windows:
+    container_name: hashtopolis_agent_windows
+    build:
+      context: ../..
+      dockerfile: .devcontainer/windows/Dockerfile
+    volumes:
+      - ../..:C:\App\

.devcontainer/windows/entrypoint.ps1

@@ -0,0 +1,2 @@
+powershell C:\fix-hosts.ps1
+cmd /c ping -t localhost > $null

.devcontainer/windows/fix-hosts.ps1

@@ -0,0 +1,43 @@
+# Source https://github.com/docker/for-win/issues/1976
+# Credits https://github.com/brunnotelma
+$hostsFile = "C:\Windows\System32\drivers\etc\hosts"
+
+try {
+    $DnsEntries = @("host.docker.internal", "gateway.docker.internal")
+    # Tries resolving names for Docker
+    foreach ($Entry in $DnsEntries) {
+        # If any of the names are not resolved, throws an exception
+        Resolve-DnsName -Name $Entry -ErrorAction Stop
+    }
+
+    # If it passes, means that DNS is already configured
+    Write-Host("DNS settings are already configured.")
+} catch {
+    # Gets the gateway IP address, that is the Host's IP address in the Docker network
+    $ip = (ipconfig | where-object {$_ -match "Default Gateway"} | foreach-object{$_.Split(":")[1]}).Trim()
+    # Read the current content from Hosts file
+    $src = [System.IO.File]::ReadAllLines($hostsFile)
+    # Add the a new line after the content
+    $lines = $src += ""
+    
+    # Check the hosts file and write it in if its not there...
+    if((cat $hostsFile | Select-String -Pattern "host.docker.internal") -And (cat $hostsFile | Select-String -Pattern "gateway.docker.internal")) {
+        For ($i=0; $i -le $lines.length; $i++) {
+            if ($lines[$i].Contains("host.docker.internal"))
+            {
+                $lines[$i] = ("{0} host.docker.internal" -f $ip)
+                $lines[$i+1] = ("{0} gateway.docker.internal" -f $ip)
+                break
+            }
+        }
+    } else {
+        $lines = $lines += "# Added by Docker for Windows"
+        $lines = $lines += ("{0} host.docker.internal" -f $ip)
+        $lines = $lines += ("{0} gateway.docker.internal" -f $ip)
+        $lines = $lines += "# End of section"
+    }
+    # Writes the new content to the Hosts file
+    [System.IO.File]::WriteAllLines($hostsFile, [string[]]$lines)
+    
+    Write-Host("New DNS settings written successfully.")
+}

.gitignore

@@ -1,8 +1,11 @@
 *.iml
 *.exe
+7zr
 *.log
 *.json
+!/tests/*.json
 crackers
+preprocessors
 prince
 files
 hashlists

.vscode/launch.json

@@ -1,4 +1,5 @@
 {
+    "version": "0.2.0",
     "configurations": [
         {
             "name": "Python: Current File",

changelog.md

@@ -1,3 +1,11 @@
+## v0.7.0 -> v0.7.1
+
+### Bugfixes
+
+* Agent working again on Windows, all paths have been converted to Pathlib.Path and using the correct quoting of arguments.
+* 7z wordlist not extracting correctly on Windows.
+* preprocessor not working correctly on both Windows and Linux.
+
 ## v0.6.1 -> v0.7.0
 
 ### Enhancements

htpclient/binarydownload.py

@@ -1,5 +1,6 @@
 import logging
 import os.path
+from pathlib import Path
 import stat
 import sys
 from time import sleep
@@ -135,7 +136,7 @@ def check_prince(self):
 
     def check_preprocessor(self, task):
         logging.debug("Checking if requested preprocessor is present...")
-        path = self.config.get_value('preprocessors-path') + "/" + str(task.get_task()['preprocessor']) + "/"
+        path = Path(self.config.get_value('preprocessors-path'), str(task.get_task()['preprocessor']))
         query = copy_and_set_token(dict_downloadBinary, self.config.get_value('token'))
         query['type'] = 'preprocessor'
         query['preprocessorId'] = task.get_task()['preprocessor']
@@ -160,20 +161,20 @@ def check_preprocessor(self, task):
                 sleep(5)
                 return False
             if Initialize.get_os() == 1:
-                os.system("7zr" + Initialize.get_os_extension() + " x -otemp temp.7z")
+                os.system(f"7zr{Initialize.get_os_extension()} x -otemp temp.7z")
             else:
-                os.system("./7zr" + Initialize.get_os_extension() + " x -otemp temp.7z")
+                os.system(f"./7zr{Initialize.get_os_extension()} x -otemp temp.7z")
             for name in os.listdir("temp"):  # this part needs to be done because it is compressed with the main subfolder of prince
-                if os.path.isdir("temp/" + name):
-                    os.rename("temp/" + name, path)
+                if os.path.isdir(Path('temp', name)):
+                    os.rename(Path('temp', name), path)
                     break
             os.unlink("temp.7z")
             os.rmdir("temp")
             logging.debug("Preprocessor downloaded and extracted")
         return True
 
     def check_version(self, cracker_id):
-        path = self.config.get_value('crackers-path') + "/" + str(cracker_id) + "/"
+        path = Path(self.config.get_value('crackers-path'), str(cracker_id))
         query = copy_and_set_token(dict_downloadBinary, self.config.get_value('token'))
         query['type'] = 'cracker'
         query['binaryVersionId'] = cracker_id
@@ -195,15 +196,28 @@ def check_version(self, cracker_id):
                     logging.error("Download of cracker binary failed!")
                     sleep(5)
                     return False
+
+                # we need to extract the 7zip
+                temp_folder = Path(self.config.get_value('crackers-path'), 'temp')
+                zip_file = Path(self.config.get_value('crackers-path'), f'{cracker_id}.7z')
+
                 if Initialize.get_os() == 1:
-                    os.system("7zr" + Initialize.get_os_extension() + " x -o'" + self.config.get_value('crackers-path') + "/temp' '" + self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z'")
+                    # Windows
+                    cmd = f'7zr{Initialize.get_os_extension()} x -o"{temp_folder}" "{zip_file}"'
                 else:
-                    os.system("./7zr" + Initialize.get_os_extension() + " x -o'" + self.config.get_value('crackers-path') + "/temp' '" + self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z'")
-                os.unlink(self.config.get_value('crackers-path') + "/" + str(cracker_id) + ".7z")
-                for name in os.listdir(self.config.get_value('crackers-path') + "/temp"):
-                    if os.path.isdir(self.config.get_value('crackers-path') + "/temp/" + name):
-                        os.rename(self.config.get_value('crackers-path') + "/temp/" + name, self.config.get_value('crackers-path') + "/" + str(cracker_id))
+                    # Linux
+                    cmd = f"./7zr{Initialize.get_os_extension()} x -o'{temp_folder}' '{zip_file}'"
+                os.system(cmd)
+                
+                # Clean up 7zip
+                os.unlink(zip_file)
+
+                # Workaround for a 7zip containing a folder name or already the contents of a cracker
+                for name in os.listdir(temp_folder):
+                    to_check_path = Path(temp_folder, name)
+                    if os.path.isdir(to_check_path):
+                        os.rename(to_check_path, path)
                     else:
-                        os.rename(self.config.get_value('crackers-path') + "/temp", self.config.get_value('crackers-path') + "/" + str(cracker_id))
+                        os.rename(temp_folder, path)
                         break
         return True

htpclient/download.py

@@ -14,8 +14,6 @@ class Download:
     def download(url, output, no_header=False):
         try:
             session = Session().s
-            if Initialize.get_os() == 1:
-                output = output.replace("/", '\\')
 
             # Check header
             if not no_header: